From e470ca89ba77b2f200ff3a8ad65c74028f42c5f7 Mon Sep 17 00:00:00 2001 From: Cyndy Ishida Date: Thu, 21 Mar 2024 15:03:34 -0700 Subject: [PATCH 001/306] [InstallAPI] Report exports discovered in binary but not in interface (#86025) This patch completes the classes of errors installapi can detect. --- .../clang/Basic/DiagnosticInstallAPIKinds.td | 1 + .../include/clang/InstallAPI/DylibVerifier.h | 13 +- clang/lib/InstallAPI/DylibVerifier.cpp | 171 ++++++- clang/test/InstallAPI/diagnostics-cpp.test | 2 + clang/test/InstallAPI/linker-symbols.test | 440 ++++++++++++++++++ .../mismatching-objc-class-symbols.test | 269 +++++++++++ clang/test/InstallAPI/symbol-flags.test | 290 ++++++++++++ .../clang-installapi/ClangInstallAPI.cpp | 2 +- llvm/lib/TextAPI/BinaryReader/DylibReader.cpp | 7 +- 9 files changed, 1174 insertions(+), 21 deletions(-) create mode 100644 clang/test/InstallAPI/linker-symbols.test create mode 100644 clang/test/InstallAPI/mismatching-objc-class-symbols.test create mode 100644 clang/test/InstallAPI/symbol-flags.test diff --git a/clang/include/clang/Basic/DiagnosticInstallAPIKinds.td b/clang/include/clang/Basic/DiagnosticInstallAPIKinds.td index f99a5fca64cb..a4c6e630ac5f 100644 --- a/clang/include/clang/Basic/DiagnosticInstallAPIKinds.td +++ b/clang/include/clang/Basic/DiagnosticInstallAPIKinds.td @@ -26,6 +26,7 @@ def warn_library_hidden_symbol : Warning<"declaration has external linkage, but def warn_header_hidden_symbol : Warning<"symbol exported in dynamic library, but marked hidden in declaration '%0'">, InGroup; def err_header_hidden_symbol : Error<"symbol exported in dynamic library, but marked hidden in declaration '%0'">; def err_header_symbol_missing : Error<"no declaration found for exported symbol '%0' in dynamic library">; +def warn_header_symbol_missing : Warning<"no declaration was found for exported symbol '%0' in dynamic library">, InGroup; def warn_header_availability_mismatch : Warning<"declaration '%0' is marked %select{available|unavailable}1," " but symbol is %select{not |}2exported in dynamic library">, InGroup; def err_header_availability_mismatch : Error<"declaration '%0' is marked %select{available|unavailable}1," diff --git a/clang/include/clang/InstallAPI/DylibVerifier.h b/clang/include/clang/InstallAPI/DylibVerifier.h index bbfa8711313e..49de24763f1f 100644 --- a/clang/include/clang/InstallAPI/DylibVerifier.h +++ b/clang/include/clang/InstallAPI/DylibVerifier.h @@ -28,7 +28,7 @@ enum class VerificationMode { /// lifetime of InstallAPI. /// As declarations are collected during AST traversal, they are /// compared as symbols against what is available in the binary dylib. -class DylibVerifier { +class DylibVerifier : llvm::MachO::RecordVisitor { private: struct SymbolContext; @@ -72,6 +72,9 @@ public: Result verify(ObjCIVarRecord *R, const FrontendAttrs *FA, const StringRef SuperClass); + // Scan through dylib slices and report any remaining missing exports. + Result verifyRemainingSymbols(); + /// Initialize target for verification. void setTarget(const Target &T); @@ -128,6 +131,14 @@ private: /// Find matching dylib slice for target triple that is being parsed. void assignSlice(const Target &T); + /// Shared implementation for verifying exported symbols in dylib. + void visitSymbolInDylib(const Record &R, SymbolContext &SymCtx); + + void visitGlobal(const GlobalRecord &R) override; + void visitObjCInterface(const ObjCInterfaceRecord &R) override; + void visitObjCCategory(const ObjCCategoryRecord &R) override; + void visitObjCIVar(const ObjCIVarRecord &R, const StringRef Super); + /// Gather annotations for symbol for error reporting. std::string getAnnotatedName(const Record *R, SymbolContext &SymCtx, bool ValidSourceLoc = true); diff --git a/clang/lib/InstallAPI/DylibVerifier.cpp b/clang/lib/InstallAPI/DylibVerifier.cpp index 24e0d0addf2f..94b8e9cd3233 100644 --- a/clang/lib/InstallAPI/DylibVerifier.cpp +++ b/clang/lib/InstallAPI/DylibVerifier.cpp @@ -66,17 +66,15 @@ std::string DylibVerifier::getAnnotatedName(const Record *R, Annotation += "(tlv) "; // Check if symbol represents only part of a @interface declaration. - const bool IsAnnotatedObjCClass = - ((SymCtx.ObjCIFKind != ObjCIFSymbolKind::None) && - (SymCtx.ObjCIFKind <= ObjCIFSymbolKind::EHType)); - - if (IsAnnotatedObjCClass) { - if (SymCtx.ObjCIFKind == ObjCIFSymbolKind::EHType) - Annotation += "Exception Type of "; - if (SymCtx.ObjCIFKind == ObjCIFSymbolKind::MetaClass) - Annotation += "Metaclass of "; - if (SymCtx.ObjCIFKind == ObjCIFSymbolKind::Class) - Annotation += "Class of "; + switch (SymCtx.ObjCIFKind) { + default: + break; + case ObjCIFSymbolKind::EHType: + return Annotation + "Exception Type of " + PrettyName; + case ObjCIFSymbolKind::MetaClass: + return Annotation + "Metaclass of " + PrettyName; + case ObjCIFSymbolKind::Class: + return Annotation + "Class of " + PrettyName; } // Only print symbol type prefix or leading "_" if there is no source location @@ -90,9 +88,6 @@ std::string DylibVerifier::getAnnotatedName(const Record *R, return Annotation + PrettyName; } - if (IsAnnotatedObjCClass) - return Annotation + PrettyName; - switch (SymCtx.Kind) { case EncodeKind::GlobalSymbol: return Annotation + PrettyName; @@ -332,9 +327,9 @@ bool DylibVerifier::compareSymbolFlags(const Record *R, SymbolContext &SymCtx, } if (!DR->isThreadLocalValue() && R->isThreadLocalValue()) { Ctx.emitDiag([&]() { - SymCtx.FA->D->getLocation(), - Ctx.Diag->Report(diag::err_header_symbol_flags_mismatch) - << getAnnotatedName(DR, SymCtx) << R->isThreadLocalValue(); + Ctx.Diag->Report(SymCtx.FA->D->getLocation(), + diag::err_header_symbol_flags_mismatch) + << getAnnotatedName(R, SymCtx) << R->isThreadLocalValue(); }); return false; } @@ -520,5 +515,147 @@ void DylibVerifier::VerifierContext::emitDiag( Report(); } +// The existence of weak-defined RTTI can not always be inferred from the +// header files because they can be generated as part of an implementation +// file. +// InstallAPI doesn't warn about weak-defined RTTI, because this doesn't affect +// static linking and so can be ignored for text-api files. +static bool shouldIgnoreCpp(StringRef Name, bool IsWeakDef) { + return (IsWeakDef && + (Name.starts_with("__ZTI") || Name.starts_with("__ZTS"))); +} +void DylibVerifier::visitSymbolInDylib(const Record &R, SymbolContext &SymCtx) { + // Undefined symbols should not be in InstallAPI generated text-api files. + if (R.isUndefined()) { + updateState(Result::Valid); + return; + } + + // Internal symbols should not be in InstallAPI generated text-api files. + if (R.isInternal()) { + updateState(Result::Valid); + return; + } + + // Allow zippered symbols with potentially mismatching availability + // between macOS and macCatalyst in the final text-api file. + const StringRef SymbolName(SymCtx.SymbolName); + if (const Symbol *Sym = Exports->findSymbol(SymCtx.Kind, SymCtx.SymbolName, + SymCtx.ObjCIFKind)) { + if (Sym->hasArchitecture(Ctx.Target.Arch)) { + updateState(Result::Ignore); + return; + } + } + + if (shouldIgnoreCpp(SymbolName, R.isWeakDefined())) { + updateState(Result::Valid); + return; + } + + // All checks at this point classify as some kind of violation that should be + // reported. + + // Regardless of verification mode, error out on mismatched special linker + // symbols. + if (SymbolName.starts_with("$ld$")) { + Ctx.emitDiag([&]() { + Ctx.Diag->Report(diag::err_header_symbol_missing) + << getAnnotatedName(&R, SymCtx, /*ValidSourceLoc=*/false); + }); + updateState(Result::Invalid); + return; + } + + // Missing declarations for exported symbols are hard errors on Pedantic mode. + if (Mode == VerificationMode::Pedantic) { + Ctx.emitDiag([&]() { + Ctx.Diag->Report(diag::err_header_symbol_missing) + << getAnnotatedName(&R, SymCtx, /*ValidSourceLoc=*/false); + }); + updateState(Result::Invalid); + return; + } + + // Missing declarations for exported symbols are warnings on ErrorsAndWarnings + // mode. + if (Mode == VerificationMode::ErrorsAndWarnings) { + Ctx.emitDiag([&]() { + Ctx.Diag->Report(diag::warn_header_symbol_missing) + << getAnnotatedName(&R, SymCtx, /*ValidSourceLoc=*/false); + }); + updateState(Result::Ignore); + return; + } + + // Missing declarations are dropped for ErrorsOnly mode. It is the last + // remaining mode. + updateState(Result::Ignore); + return; +} + +void DylibVerifier::visitGlobal(const GlobalRecord &R) { + if (R.isVerified()) + return; + SymbolContext SymCtx; + SimpleSymbol Sym = parseSymbol(R.getName()); + SymCtx.SymbolName = Sym.Name; + SymCtx.Kind = Sym.Kind; + visitSymbolInDylib(R, SymCtx); +} + +void DylibVerifier::visitObjCIVar(const ObjCIVarRecord &R, + const StringRef Super) { + if (R.isVerified()) + return; + SymbolContext SymCtx; + SymCtx.SymbolName = ObjCIVarRecord::createScopedName(Super, R.getName()); + SymCtx.Kind = EncodeKind::ObjectiveCInstanceVariable; + visitSymbolInDylib(R, SymCtx); +} + +void DylibVerifier::visitObjCInterface(const ObjCInterfaceRecord &R) { + if (R.isVerified()) + return; + SymbolContext SymCtx; + SymCtx.SymbolName = R.getName(); + SymCtx.ObjCIFKind = assignObjCIFSymbolKind(&R); + if (SymCtx.ObjCIFKind > ObjCIFSymbolKind::EHType) { + if (R.hasExceptionAttribute()) { + SymCtx.Kind = EncodeKind::ObjectiveCClassEHType; + visitSymbolInDylib(R, SymCtx); + } + SymCtx.Kind = EncodeKind::ObjectiveCClass; + visitSymbolInDylib(R, SymCtx); + } else { + SymCtx.Kind = R.hasExceptionAttribute() ? EncodeKind::ObjectiveCClassEHType + : EncodeKind::ObjectiveCClass; + visitSymbolInDylib(R, SymCtx); + } + + for (const ObjCIVarRecord *IV : R.getObjCIVars()) + visitObjCIVar(*IV, R.getName()); +} + +void DylibVerifier::visitObjCCategory(const ObjCCategoryRecord &R) { + for (const ObjCIVarRecord *IV : R.getObjCIVars()) + visitObjCIVar(*IV, R.getSuperClassName()); +} + +DylibVerifier::Result DylibVerifier::verifyRemainingSymbols() { + if (getState() == Result::NoVerify) + return Result::NoVerify; + assert(!Dylib.empty() && "No binary to verify against"); + + Ctx.DiscoveredFirstError = false; + Ctx.PrintArch = true; + for (std::shared_ptr Slice : Dylib) { + Ctx.Target = Slice->getTarget(); + Ctx.DylibSlice = Slice.get(); + Slice->visit(*this); + } + return getState(); +} + } // namespace installapi } // namespace clang diff --git a/clang/test/InstallAPI/diagnostics-cpp.test b/clang/test/InstallAPI/diagnostics-cpp.test index 658886537507..51cca129ea0a 100644 --- a/clang/test/InstallAPI/diagnostics-cpp.test +++ b/clang/test/InstallAPI/diagnostics-cpp.test @@ -21,6 +21,8 @@ CHECK-NEXT: CPP.h:5:7: error: declaration has external linkage, but symbol has i CHECK-NEXT: CPP.h:6:7: error: dynamic library symbol '(weak-def) Bar::init()' is weak defined, but its declaration is not CHECK-NEXT: int init(); CHECK-NEXT: ^ +CHECK-NEXT: warning: violations found for arm64 +CHECK-NEXT: error: no declaration found for exported symbol 'int foo(unsigned int)' in dynamic library //--- inputs.json.in { diff --git a/clang/test/InstallAPI/linker-symbols.test b/clang/test/InstallAPI/linker-symbols.test new file mode 100644 index 000000000000..1e4ddf9c45d5 --- /dev/null +++ b/clang/test/InstallAPI/linker-symbols.test @@ -0,0 +1,440 @@ +; RUN: rm -rf %t +; RUN: split-file %s %t +; RUN: sed -e "s|DSTROOT|%/t|g" %t/inputs.json.in > %t/inputs.json + +; RUN: yaml2obj %t/MagicSymbols.yaml -o %t/MagicSymbols + +; RUN: not clang-installapi -target x86_64-apple-macosx13 \ +; RUN: -install_name \ +; RUN: /System/Library/Frameworks/SpecialLinkerSymbols.framework/Versions/A/SpecialLinkerSymbols \ +; RUN: -current_version 1 -compatibility_version 1 \ +; RUN: %t/inputs.json -o %t/output.tbd \ +; RUN: --verify-mode=ErrorsOnly \ +; RUN: --verify-against=%t/MagicSymbols 2>&1 | FileCheck %s + +CHECK: warning: violations found for x86_64 +CHECK: error: no declaration found for exported symbol '$ld$add$os10.4$_symbol2' in dynamic library +CHECK: error: no declaration found for exported symbol '$ld$add$os10.5$_symbol2' in dynamic library +CHECK: error: no declaration found for exported symbol '$ld$hide$os10.6$_symbol1' in dynamic library +CHECK: error: no declaration found for exported symbol '$ld$hide$os10.7$_symbol1' in dynamic library +CHECK: error: no declaration found for exported symbol '$ld$weak$os10.5$_symbol3' in dynamic library +CHECK: error: no declaration found for exported symbol '$ld$weak$os10.4$_symbol3' in dynamic library +CHECK: error: no declaration found for exported symbol '$ld$install_name$os10.4$/System/Library/Frameworks/A.framework/Versions/A/A' in dynamic library +CHECK: error: no declaration found for exported symbol '$ld$install_name$os10.5$/System/Library/Frameworks/B.framework/Versions/A/B' in dynamic library + +;--- MagicSymbols.h +#ifndef SPECIAL_LINKER_SYMBOLS_H +#define SPECIAL_LINKER_SYMBOLS_H + +extern const int SpecialLinkerSymbolsVersion; + +extern int symbol1; +extern int symbol3; + +#endif // SPECIAL_LINKER_SYMBOLS_H + +;--- inputs.json.in +{ + "headers": [ { + "path" : "DSTROOT/MagicSymbols.h", + "type" : "project" + } + ], + "version": "3" +} + +;--- MagicSymbols.yaml +--- !mach-o +FileHeader: + magic: 0xFEEDFACF + cputype: 0x1000007 + cpusubtype: 0x3 + filetype: 0x6 + ncmds: 12 + sizeofcmds: 952 + flags: 0x100085 + reserved: 0x0 +LoadCommands: + - cmd: LC_SEGMENT_64 + cmdsize: 232 + segname: __TEXT + vmaddr: 0 + vmsize: 4096 + fileoff: 0 + filesize: 4096 + maxprot: 5 + initprot: 5 + nsects: 2 + flags: 0 + Sections: + - sectname: __text + segname: __TEXT + addr: 0xBD8 + size: 0 + offset: 0xBD8 + align: 0 + reloff: 0x0 + nreloc: 0 + flags: 0x80000000 + reserved1: 0x0 + reserved2: 0x0 + reserved3: 0x0 + content: '' + - sectname: __const + segname: __TEXT + addr: 0xBD8 + size: 4 + offset: 0xBD8 + align: 2 + reloff: 0x0 + nreloc: 0 + flags: 0x0 + reserved1: 0x0 + reserved2: 0x0 + reserved3: 0x0 + content: '07000000' + - cmd: LC_SEGMENT_64 + cmdsize: 232 + segname: __DATA + vmaddr: 4096 + vmsize: 4096 + fileoff: 4096 + filesize: 4096 + maxprot: 3 + initprot: 3 + nsects: 2 + flags: 0 + Sections: + - sectname: __data + segname: __DATA + addr: 0x1000 + size: 8 + offset: 0x1000 + align: 2 + reloff: 0x0 + nreloc: 0 + flags: 0x0 + reserved1: 0x0 + reserved2: 0x0 + reserved3: 0x0 + content: 4D00000009030000 + - sectname: __common + segname: __DATA + addr: 0x1008 + size: 8 + offset: 0x0 + align: 0 + reloff: 0x0 + nreloc: 0 + flags: 0x1 + reserved1: 0x0 + reserved2: 0x0 + reserved3: 0x0 + - cmd: LC_SEGMENT_64 + cmdsize: 72 + segname: __LINKEDIT + vmaddr: 8192 + vmsize: 944 + fileoff: 8192 + filesize: 944 + maxprot: 1 + initprot: 1 + nsects: 0 + flags: 0 + - cmd: LC_DYLD_INFO_ONLY + cmdsize: 48 + rebase_off: 0 + rebase_size: 0 + bind_off: 0 + bind_size: 0 + weak_bind_off: 0 + weak_bind_size: 0 + lazy_bind_off: 0 + lazy_bind_size: 0 + export_off: 8192 + export_size: 376 + - cmd: LC_SYMTAB + cmdsize: 24 + symoff: 8576 + nsyms: 12 + stroff: 8768 + strsize: 368 + - cmd: LC_DYSYMTAB + cmdsize: 80 + ilocalsym: 0 + nlocalsym: 0 + iextdefsym: 0 + nextdefsym: 11 + iundefsym: 11 + nundefsym: 1 + tocoff: 0 + ntoc: 0 + modtaboff: 0 + nmodtab: 0 + extrefsymoff: 0 + nextrefsyms: 0 + indirectsymoff: 0 + nindirectsyms: 0 + extreloff: 0 + nextrel: 0 + locreloff: 0 + nlocrel: 0 + - cmd: LC_ID_DYLIB + cmdsize: 120 + dylib: + name: 24 + timestamp: 0 + current_version: 65536 + compatibility_version: 65536 + Content: '/System/Library/Frameworks/SpecialLinkerSymbols.framework/Versions/A/SpecialLinkerSymbols' + ZeroPadBytes: 7 + - cmd: LC_UUID + cmdsize: 24 + uuid: 4C4C4478-5555-3144-A106-356C3C9DACA3 + - cmd: LC_BUILD_VERSION + cmdsize: 32 + platform: 1 + minos: 851968 + sdk: 983040 + ntools: 1 + Tools: + - tool: 4 + version: 1245184 + - cmd: LC_LOAD_DYLIB + cmdsize: 56 + dylib: + name: 24 + timestamp: 0 + current_version: 88539136 + compatibility_version: 65536 + Content: '/usr/lib/libSystem.B.dylib' + ZeroPadBytes: 6 + - cmd: LC_FUNCTION_STARTS + cmdsize: 16 + dataoff: 8568 + datasize: 8 + - cmd: LC_DATA_IN_CODE + cmdsize: 16 + dataoff: 8576 + datasize: 0 +LinkEditData: + ExportTrie: + TerminalSize: 0 + NodeOffset: 0 + Name: '' + Flags: 0x0 + Address: 0x0 + Other: 0x0 + ImportName: '' + Children: + - TerminalSize: 0 + NodeOffset: 11 + Name: _ + Flags: 0x0 + Address: 0x0 + Other: 0x0 + ImportName: '' + Children: + - TerminalSize: 3 + NodeOffset: 50 + Name: SpecialLinkerSymbolsVersion + Flags: 0x0 + Address: 0xBD8 + Other: 0x0 + ImportName: '' + - TerminalSize: 0 + NodeOffset: 55 + Name: symbol + Flags: 0x0 + Address: 0x0 + Other: 0x0 + ImportName: '' + Children: + - TerminalSize: 3 + NodeOffset: 63 + Name: '3' + Flags: 0x0 + Address: 0x1004 + Other: 0x0 + ImportName: '' + - TerminalSize: 3 + NodeOffset: 68 + Name: '1' + Flags: 0x0 + Address: 0x1000 + Other: 0x0 + ImportName: '' + - TerminalSize: 0 + NodeOffset: 73 + Name: '$ld$' + Flags: 0x0 + Address: 0x0 + Other: 0x0 + ImportName: '' + Children: + - TerminalSize: 0 + NodeOffset: 134 + Name: 'add$os10.' + Flags: 0x0 + Address: 0x0 + Other: 0x0 + ImportName: '' + Children: + - TerminalSize: 3 + NodeOffset: 162 + Name: '4$_symbol2' + Flags: 0x0 + Address: 0x1008 + Other: 0x0 + ImportName: '' + - TerminalSize: 3 + NodeOffset: 167 + Name: '5$_symbol2' + Flags: 0x0 + Address: 0x1009 + Other: 0x0 + ImportName: '' + - TerminalSize: 0 + NodeOffset: 172 + Name: 'hide$os10.' + Flags: 0x0 + Address: 0x0 + Other: 0x0 + ImportName: '' + Children: + - TerminalSize: 3 + NodeOffset: 200 + Name: '6$_symbol1' + Flags: 0x0 + Address: 0x100A + Other: 0x0 + ImportName: '' + - TerminalSize: 3 + NodeOffset: 205 + Name: '7$_symbol1' + Flags: 0x0 + Address: 0x100B + Other: 0x0 + ImportName: '' + - TerminalSize: 0 + NodeOffset: 210 + Name: 'weak$os10.' + Flags: 0x0 + Address: 0x0 + Other: 0x0 + ImportName: '' + Children: + - TerminalSize: 3 + NodeOffset: 238 + Name: '5$_symbol3' + Flags: 0x0 + Address: 0x100F + Other: 0x0 + ImportName: '' + - TerminalSize: 3 + NodeOffset: 243 + Name: '4$_symbol3' + Flags: 0x0 + Address: 0x100E + Other: 0x0 + ImportName: '' + - TerminalSize: 0 + NodeOffset: 248 + Name: 'install_name$os10.' + Flags: 0x0 + Address: 0x0 + Other: 0x0 + ImportName: '' + Children: + - TerminalSize: 3 + NodeOffset: 362 + Name: '4$/System/Library/Frameworks/A.framework/Versions/A/A' + Flags: 0x0 + Address: 0x100C + Other: 0x0 + ImportName: '' + - TerminalSize: 3 + NodeOffset: 367 + Name: '5$/System/Library/Frameworks/B.framework/Versions/A/B' + Flags: 0x0 + Address: 0x100D + Other: 0x0 + ImportName: '' + NameList: + - n_strx: 2 + n_type: 0xF + n_sect: 4 + n_desc: 0 + n_value: 4104 + - n_strx: 26 + n_type: 0xF + n_sect: 4 + n_desc: 0 + n_value: 4105 + - n_strx: 50 + n_type: 0xF + n_sect: 4 + n_desc: 0 + n_value: 4106 + - n_strx: 75 + n_type: 0xF + n_sect: 4 + n_desc: 0 + n_value: 4107 + - n_strx: 100 + n_type: 0xF + n_sect: 4 + n_desc: 0 + n_value: 4108 + - n_strx: 176 + n_type: 0xF + n_sect: 4 + n_desc: 0 + n_value: 4109 + - n_strx: 252 + n_type: 0xF + n_sect: 4 + n_desc: 0 + n_value: 4110 + - n_strx: 277 + n_type: 0xF + n_sect: 4 + n_desc: 0 + n_value: 4111 + - n_strx: 302 + n_type: 0xF + n_sect: 2 + n_desc: 0 + n_value: 3032 + - n_strx: 331 + n_type: 0xF + n_sect: 3 + n_desc: 0 + n_value: 4096 + - n_strx: 340 + n_type: 0xF + n_sect: 3 + n_desc: 0 + n_value: 4100 + - n_strx: 349 + n_type: 0x1 + n_sect: 0 + n_desc: 256 + n_value: 0 + StringTable: + - ' ' + - '$ld$add$os10.4$_symbol2' + - '$ld$add$os10.5$_symbol2' + - '$ld$hide$os10.6$_symbol1' + - '$ld$hide$os10.7$_symbol1' + - '$ld$install_name$os10.4$/System/Library/Frameworks/A.framework/Versions/A/A' + - '$ld$install_name$os10.5$/System/Library/Frameworks/B.framework/Versions/A/B' + - '$ld$weak$os10.4$_symbol3' + - '$ld$weak$os10.5$_symbol3' + - _SpecialLinkerSymbolsVersion + - _symbol1 + - _symbol3 + - dyld_stub_binder + - '' + - '' +... diff --git a/clang/test/InstallAPI/mismatching-objc-class-symbols.test b/clang/test/InstallAPI/mismatching-objc-class-symbols.test new file mode 100644 index 000000000000..3b4acf1035ac --- /dev/null +++ b/clang/test/InstallAPI/mismatching-objc-class-symbols.test @@ -0,0 +1,269 @@ +; RUN: rm -rf %t +; RUN: split-file %s %t +; RUN: sed -e "s|DSTROOT|%/t|g" %t/inputs.json.in > %t/inputs.json +; RUN: yaml2obj %t/swift-objc-class.yaml -o %t/libswift-objc.dylib + +// Try out dylib that only has 1 symbol for a ObjCClass, with no declarations in header. +; RUN: clang-installapi -target arm64-apple-macos14 -dynamiclib \ +; RUN: -install_name tmp.dylib --verify-against=%t/libswift-objc.dylib \ +; RUN: -I%t/usr/include %t/inputs.json -o %t/missing.tbd \ +; RUN: --verify-mode=ErrorsAndWarnings 2>&1 | FileCheck --check-prefix MISSING_DECL %s +; RUN: llvm-readtapi --compare %t/missing.tbd %t/missing-expected.tbd + +// Try out a dylib that only has 1 symbol for a ObjCClass, +// but a complete ObjCClass decl in header. +; RUN: clang-installapi -target arm64-apple-macos14 -dynamiclib \ +; RUN: -install_name tmp.dylib --verify-against=%t/libswift-objc.dylib \ +; RUN: -I%t/usr/include %t/inputs.json -o %t/mismatching.tbd \ +; RUN: --verify-mode=Pedantic -DFULL_DECL 2>&1 | FileCheck --check-prefix MISMATCH_DECL %s +; RUN: llvm-readtapi -compare %t/mismatching.tbd %t/mismatching-expected.tbd + +// Try out a dylib that only has 1 symbol for a ObjCClass, but is represented in header. +; RUN: clang-installapi -target arm64-apple-macos14 \ +; RUN: -install_name tmp.dylib --verify-against=%t/libswift-objc.dylib \ +; RUN: -I%t/usr/include %t/inputs.json -o %t/matching.tbd \ +; RUN: --verify-mode=Pedantic \ +; RUN: -DHAS_META_DECL 2>&1 | FileCheck --allow-empty %s + +; MISSING_DECL: violations found for arm64 +; MISSING_DECL-NEXT: warning: no declaration was found for exported symbol 'Metaclass of Suggestion' in dynamic library + +; MISMATCH_DECL: violations found for arm64-apple-macos14 +; MISMATCH_DECL: warning: declaration has external linkage, but dynamic library doesn't have symbol 'Class of Suggestion' + +; CHECK-NOT: error +; CHECK-NOT: warning + + +;--- usr/include/mismatch.h +#if HAS_META_DECL +int metaclass __asm("_OBJC_METACLASS_$_Suggestion"); +#endif + +#if FULL_DECL +@interface Suggestion +@end +#endif + +;--- inputs.json.in +{ + "headers": [ { + "path" : "DSTROOT/usr/include/mismatch.h", + "type" : "public" + } + ], + "version": "3" +} + +;--- missing-expected.tbd +--- !tapi-tbd +tbd-version: 4 +targets: [ arm64-macos ] +flags: [ not_app_extension_safe ] +install-name: tmp.dylib +current-version: 0 +compatibility-version: 0 +... + +;--- mismatching-expected.tbd +--- !tapi-tbd +tbd-version: 4 +targets: [ arm64-macos ] +flags: [ not_app_extension_safe ] +install-name: tmp.dylib +current-version: 0 +compatibility-version: 0 +exports: + - targets: [ arm64-macos ] + objc-classes: [ Suggestion ] +... + +;--- swift-objc-class.yaml +--- !mach-o +FileHeader: + magic: 0xFEEDFACF + cputype: 0x100000C + cpusubtype: 0x0 + filetype: 0x6 + ncmds: 13 + sizeofcmds: 752 + flags: 0x100085 + reserved: 0x0 +LoadCommands: + - cmd: LC_SEGMENT_64 + cmdsize: 232 + segname: __TEXT + vmaddr: 0 + vmsize: 16384 + fileoff: 0 + filesize: 16384 + maxprot: 5 + initprot: 5 + nsects: 2 + flags: 0 + Sections: + - sectname: __text + segname: __TEXT + addr: 0x330 + size: 0 + offset: 0x330 + align: 0 + reloff: 0x0 + nreloc: 0 + flags: 0x80000000 + reserved1: 0x0 + reserved2: 0x0 + reserved3: 0x0 + content: '' + - sectname: __const + segname: __TEXT + addr: 0x330 + size: 1 + offset: 0x330 + align: 0 + reloff: 0x0 + nreloc: 0 + flags: 0x0 + reserved1: 0x0 + reserved2: 0x0 + reserved3: 0x0 + content: '61' + - cmd: LC_SEGMENT_64 + cmdsize: 72 + segname: __LINKEDIT + vmaddr: 16384 + vmsize: 416 + fileoff: 16384 + filesize: 416 + maxprot: 1 + initprot: 1 + nsects: 0 + flags: 0 + - cmd: LC_DYLD_INFO_ONLY + cmdsize: 48 + rebase_off: 0 + rebase_size: 0 + bind_off: 0 + bind_size: 0 + weak_bind_off: 0 + weak_bind_size: 0 + lazy_bind_off: 0 + lazy_bind_size: 0 + export_off: 16384 + export_size: 40 + - cmd: LC_SYMTAB + cmdsize: 24 + symoff: 16432 + nsyms: 2 + stroff: 16464 + strsize: 48 + - cmd: LC_DYSYMTAB + cmdsize: 80 + ilocalsym: 0 + nlocalsym: 0 + iextdefsym: 0 + nextdefsym: 1 + iundefsym: 1 + nundefsym: 1 + tocoff: 0 + ntoc: 0 + modtaboff: 0 + nmodtab: 0 + extrefsymoff: 0 + nextrefsyms: 0 + indirectsymoff: 0 + nindirectsyms: 0 + extreloff: 0 + nextrel: 0 + locreloff: 0 + nlocrel: 0 + - cmd: LC_ID_DYLIB + cmdsize: 40 + dylib: + name: 24 + timestamp: 0 + current_version: 0 + compatibility_version: 0 + Content: tmp.dylib + ZeroPadBytes: 7 + - cmd: LC_UUID + cmdsize: 24 + uuid: 4C4C4443-5555-3144-A142-97179769CBE0 + - cmd: LC_BUILD_VERSION + cmdsize: 32 + platform: 1 + minos: 917504 + sdk: 983040 + ntools: 1 + Tools: + - tool: 4 + version: 1245184 + - cmd: LC_LOAD_DYLIB + cmdsize: 96 + dylib: + name: 24 + timestamp: 0 + current_version: 197656576 + compatibility_version: 19660800 + Content: '/System/Library/Frameworks/Foundation.framework/Versions/C/Foundation' + ZeroPadBytes: 3 + - cmd: LC_LOAD_DYLIB + cmdsize: 56 + dylib: + name: 24 + timestamp: 0 + current_version: 88473600 + compatibility_version: 65536 + Content: '/usr/lib/libSystem.B.dylib' + ZeroPadBytes: 6 + - cmd: LC_FUNCTION_STARTS + cmdsize: 16 + dataoff: 16424 + datasize: 8 + - cmd: LC_DATA_IN_CODE + cmdsize: 16 + dataoff: 16432 + datasize: 0 + - cmd: LC_CODE_SIGNATURE + cmdsize: 16 + dataoff: 16512 + datasize: 288 +LinkEditData: + ExportTrie: + TerminalSize: 0 + NodeOffset: 0 + Name: '' + Flags: 0x0 + Address: 0x0 + Other: 0x0 + ImportName: '' + Children: + - TerminalSize: 3 + NodeOffset: 32 + Name: '_OBJC_METACLASS_$_Suggestion' + Flags: 0x0 + Address: 0x330 + Other: 0x0 + ImportName: '' + NameList: + - n_strx: 2 + n_type: 0xF + n_sect: 2 + n_desc: 0 + n_value: 816 + - n_strx: 31 + n_type: 0x1 + n_sect: 0 + n_desc: 512 + n_value: 0 + StringTable: + - ' ' + - '_OBJC_METACLASS_$_Suggestion' + - dyld_stub_binder + FunctionStarts: [ 0x330 ] +... +// Generated from: +// xcrun -sdk macosx clang tmp.c -dynamiclib -install_name tmp.dylib +// tmp.c: +// __attribute__((visibility("default"))) +// const char Meta __asm("_OBJC_METACLASS_$_Suggestion") = 'a'; diff --git a/clang/test/InstallAPI/symbol-flags.test b/clang/test/InstallAPI/symbol-flags.test new file mode 100644 index 000000000000..3f68afd17e3b --- /dev/null +++ b/clang/test/InstallAPI/symbol-flags.test @@ -0,0 +1,290 @@ +; RUN: rm -rf %t +; RUN: split-file %s %t +; RUN: sed -e "s|DSTROOT|%/t|g" %t/inputs.json.in > %t/inputs.json + +; RUN: yaml2obj %t/flags.yaml -o %t/SymbolFlags + +; RUN: not clang-installapi -x c++ --target=arm64-apple-macos13 \ +; RUN: -install_name /System/Library/Frameworks/SymbolFlags.framework/Versions/A/SymbolFlags \ +; RUN: -current_version 1 -compatibility_version 1 \ +; RUN: %t/inputs.json -o output.tbd \ +; RUN: --verify-against=%t/SymbolFlags \ +; RUN: --verify-mode=ErrorsOnly 2>&1 | FileCheck %s + +; CHECK: project.h:2:21: error: declaration '(tlv) val' is thread local, but symbol is not in dynamic library +; CHECK-NEXT: extern __thread int val; +; CHECK: project.h:3:13: error: dynamic library symbol '(weak-def) __Z12my_weak_funcv' is weak defined, but its declaration is not +; CHECK-NEXT: extern void my_weak_func(); + +;--- project.h +extern void my_func(); +extern __thread int val; +extern void my_weak_func(); + +;--- inputs.json.in +{ + "headers": [ { + "path" : "DSTROOT/project.h", + "type" : "project" + } + ], + "version": "3" +} + +;--- flags.yaml +--- !mach-o +FileHeader: + magic: 0xFEEDFACF + cputype: 0x100000C + cpusubtype: 0x0 + filetype: 0x6 + ncmds: 14 + sizeofcmds: 912 + flags: 0x118085 + reserved: 0x0 +LoadCommands: + - cmd: LC_SEGMENT_64 + cmdsize: 232 + segname: __TEXT + vmaddr: 0 + vmsize: 16384 + fileoff: 0 + filesize: 16384 + maxprot: 5 + initprot: 5 + nsects: 2 + flags: 0 + Sections: + - sectname: __text + segname: __TEXT + addr: 0xFB0 + size: 8 + offset: 0xFB0 + align: 2 + reloff: 0x0 + nreloc: 0 + flags: 0x80000400 + reserved1: 0x0 + reserved2: 0x0 + reserved3: 0x0 + content: C0035FD6C0035FD6 + - sectname: __unwind_info + segname: __TEXT + addr: 0xFB8 + size: 4152 + offset: 0xFB8 + align: 2 + reloff: 0x0 + nreloc: 0 + flags: 0x0 + reserved1: 0x0 + reserved2: 0x0 + reserved3: 0x0 + content: 010000001C000000010000002000000000000000200000000200000000000002B00F00003800000038000000B80F00000000000038000000030000000C0001001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 + - cmd: LC_SEGMENT_64 + cmdsize: 152 + segname: __DATA + vmaddr: 16384 + vmsize: 16384 + fileoff: 16384 + filesize: 0 + maxprot: 3 + initprot: 3 + nsects: 1 + flags: 0 + Sections: + - sectname: __common + segname: __DATA + addr: 0x4000 + size: 4 + offset: 0x0 + align: 2 + reloff: 0x0 + nreloc: 0 + flags: 0x1 + reserved1: 0x0 + reserved2: 0x0 + reserved3: 0x0 + - cmd: LC_SEGMENT_64 + cmdsize: 72 + segname: __LINKEDIT + vmaddr: 32768 + vmsize: 480 + fileoff: 16384 + filesize: 480 + maxprot: 1 + initprot: 1 + nsects: 0 + flags: 0 + - cmd: LC_DYLD_INFO_ONLY + cmdsize: 48 + rebase_off: 0 + rebase_size: 0 + bind_off: 0 + bind_size: 0 + weak_bind_off: 0 + weak_bind_size: 0 + lazy_bind_off: 0 + lazy_bind_size: 0 + export_off: 16384 + export_size: 64 + - cmd: LC_SYMTAB + cmdsize: 24 + symoff: 16456 + nsyms: 4 + stroff: 16520 + strsize: 56 + - cmd: LC_DYSYMTAB + cmdsize: 80 + ilocalsym: 0 + nlocalsym: 0 + iextdefsym: 0 + nextdefsym: 3 + iundefsym: 3 + nundefsym: 1 + tocoff: 0 + ntoc: 0 + modtaboff: 0 + nmodtab: 0 + extrefsymoff: 0 + nextrefsyms: 0 + indirectsymoff: 0 + nindirectsyms: 0 + extreloff: 0 + nextrel: 0 + locreloff: 0 + nlocrel: 0 + - cmd: LC_ID_DYLIB + cmdsize: 96 + dylib: + name: 24 + timestamp: 0 + current_version: 65536 + compatibility_version: 65536 + Content: '/System/Library/Frameworks/SymbolFlags.framework/Versions/A/SymbolFlags' + ZeroPadBytes: 1 + - cmd: LC_UUID + cmdsize: 24 + uuid: 4C4C4436-5555-3144-A1AF-5D3063ACFC99 + - cmd: LC_BUILD_VERSION + cmdsize: 32 + platform: 1 + minos: 851968 + sdk: 983040 + ntools: 1 + Tools: + - tool: 4 + version: 1245184 + - cmd: LC_LOAD_DYLIB + cmdsize: 48 + dylib: + name: 24 + timestamp: 0 + current_version: 117985024 + compatibility_version: 65536 + Content: '/usr/lib/libc++.1.dylib' + ZeroPadBytes: 1 + - cmd: LC_LOAD_DYLIB + cmdsize: 56 + dylib: + name: 24 + timestamp: 0 + current_version: 88473600 + compatibility_version: 65536 + Content: '/usr/lib/libSystem.B.dylib' + ZeroPadBytes: 6 + - cmd: LC_FUNCTION_STARTS + cmdsize: 16 + dataoff: 16448 + datasize: 8 + - cmd: LC_DATA_IN_CODE + cmdsize: 16 + dataoff: 16456 + datasize: 0 + - cmd: LC_CODE_SIGNATURE + cmdsize: 16 + dataoff: 16576 + datasize: 288 +LinkEditData: + ExportTrie: + TerminalSize: 0 + NodeOffset: 0 + Name: '' + Flags: 0x0 + Address: 0x0 + Other: 0x0 + ImportName: '' + Children: + - TerminalSize: 0 + NodeOffset: 5 + Name: _ + Flags: 0x0 + Address: 0x0 + Other: 0x0 + ImportName: '' + Children: + - TerminalSize: 4 + NodeOffset: 16 + Name: val + Flags: 0x0 + Address: 0x4000 + Other: 0x0 + ImportName: '' + - TerminalSize: 0 + NodeOffset: 22 + Name: _Z + Flags: 0x0 + Address: 0x0 + Other: 0x0 + ImportName: '' + Children: + - TerminalSize: 3 + NodeOffset: 52 + Name: 7my_funcv + Flags: 0x0 + Address: 0xFB0 + Other: 0x0 + ImportName: '' + - TerminalSize: 3 + NodeOffset: 57 + Name: 12my_weak_funcv + Flags: 0x4 + Address: 0xFB4 + Other: 0x0 + ImportName: '' + NameList: + - n_strx: 2 + n_type: 0xF + n_sect: 1 + n_desc: 0 + n_value: 4016 + - n_strx: 15 + n_type: 0xF + n_sect: 1 + n_desc: 128 + n_value: 4020 + - n_strx: 34 + n_type: 0xF + n_sect: 3 + n_desc: 0 + n_value: 16384 + - n_strx: 39 + n_type: 0x1 + n_sect: 0 + n_desc: 512 + n_value: 0 + StringTable: + - ' ' + - __Z7my_funcv + - __Z12my_weak_funcv + - _val + - dyld_stub_binder + FunctionStarts: [ 0xFB0, 0xFB4 ] +... + +/// Generated from: +// clang++ -mtargetos=macosx13 -arch arm64 flags.cpp +// flags.cpp: +// __attribute__((visibility("default"))) void my_func() {} +// __attribute__((weak)) void my_weak_func() {} +// int val = 0; diff --git a/clang/tools/clang-installapi/ClangInstallAPI.cpp b/clang/tools/clang-installapi/ClangInstallAPI.cpp index 54e82d78d4d2..13061cfa36ee 100644 --- a/clang/tools/clang-installapi/ClangInstallAPI.cpp +++ b/clang/tools/clang-installapi/ClangInstallAPI.cpp @@ -123,7 +123,7 @@ static bool run(ArrayRef Args, const char *ProgName) { } } - if (Ctx.Verifier->getState() == DylibVerifier::Result::Invalid) + if (Ctx.Verifier->verifyRemainingSymbols() == DylibVerifier::Result::Invalid) return EXIT_FAILURE; // After symbols have been collected, prepare to write output. diff --git a/llvm/lib/TextAPI/BinaryReader/DylibReader.cpp b/llvm/lib/TextAPI/BinaryReader/DylibReader.cpp index 0694d8f28df6..2e36d4a8b98c 100644 --- a/llvm/lib/TextAPI/BinaryReader/DylibReader.cpp +++ b/llvm/lib/TextAPI/BinaryReader/DylibReader.cpp @@ -293,8 +293,11 @@ static Error readSymbols(MachOObjectFile *Obj, RecordsSlice &Slice, RecordLinkage Linkage = RecordLinkage::Unknown; SymbolFlags RecordFlags = SymbolFlags::None; - if (Opt.Undefineds && (Flags & SymbolRef::SF_Undefined)) { - Linkage = RecordLinkage::Undefined; + if (Flags & SymbolRef::SF_Undefined) { + if (Opt.Undefineds) + Linkage = RecordLinkage::Undefined; + else + continue; if (Flags & SymbolRef::SF_Weak) RecordFlags |= SymbolFlags::WeakReferenced; } else if (Flags & SymbolRef::SF_Exported) { -- GitLab From b8e53630f899ddb8a2ec0d37bcb86608d58c4960 Mon Sep 17 00:00:00 2001 From: Alexander Richardson Date: Thu, 21 Mar 2024 15:04:19 -0700 Subject: [PATCH 002/306] [compiler-rt] Avoid pulling in __cxa_pure_virtual When building optimized versions of the runtime libraries the compiler is generally able to elide these references, but when building them for maximum debug info (with -O0), these references remain which causes the test suite to fail for tests that do not pull in the C++ standard library. Reviewed By: vitalybuka Pull Request: https://github.com/llvm/llvm-project/pull/84613 --- .../sanitizer_stacktrace_printer.h | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.h b/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.h index 10361a320344..e39cb891575e 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.h @@ -30,10 +30,15 @@ class StackTracePrinter { virtual void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no, uptr address, const AddressInfo *info, - bool vs_style, - const char *strip_path_prefix = "") = 0; + bool vs_style, const char *strip_path_prefix = "") { + // Should be pure virtual, but we can't depend on __cxa_pure_virtual. + UNIMPLEMENTED(); + } - virtual bool RenderNeedsSymbolization(const char *format) = 0; + virtual bool RenderNeedsSymbolization(const char *format) { + // Should be pure virtual, but we can't depend on __cxa_pure_virtual. + UNIMPLEMENTED(); + } void RenderSourceLocation(InternalScopedString *buffer, const char *file, int line, int column, bool vs_style, @@ -44,7 +49,10 @@ class StackTracePrinter { const char *strip_path_prefix); virtual void RenderData(InternalScopedString *buffer, const char *format, const DataInfo *DI, - const char *strip_path_prefix = "") = 0; + const char *strip_path_prefix = "") { + // Should be pure virtual, but we can't depend on __cxa_pure_virtual. + UNIMPLEMENTED(); + } private: // To be called from StackTracePrinter::GetOrInit -- GitLab From c56211b2430cf63ba3a469a4ae89cf2e829e9332 Mon Sep 17 00:00:00 2001 From: Roland McGrath Date: Thu, 21 Mar 2024 15:11:31 -0700 Subject: [PATCH 003/306] [libc] Make math-macros.h C++-friendly (#86206) The isfinite, isnan, and isinf "functions" are specified by C99..C23 to be macros that act as type-generic functions. Defining them as their __builtin_* counterparts works fine for this. However, in C++ the identifiers need to be usable in different contexts, such as being declared inside a C++ namespace. So define inline constexpr template functions for them under `#ifdef __cplusplus`. --- libc/include/llvm-libc-macros/math-macros.h | 32 ++++++++++++++++++--- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/libc/include/llvm-libc-macros/math-macros.h b/libc/include/llvm-libc-macros/math-macros.h index db8a4ea65bd6..2605535b927d 100644 --- a/libc/include/llvm-libc-macros/math-macros.h +++ b/libc/include/llvm-libc-macros/math-macros.h @@ -30,10 +30,6 @@ #define FP_LLOGB0 (-LONG_MAX - 1) #define FP_LLOGBNAN LONG_MAX -#define isfinite(x) __builtin_isfinite(x) -#define isinf(x) __builtin_isinf(x) -#define isnan(x) __builtin_isnan(x) - #ifdef __FAST_MATH__ #define math_errhandling 0 #elif defined(__NO_MATH_ERRNO__) @@ -44,4 +40,32 @@ #define math_errhandling (MATH_ERRNO | MATH_ERREXCEPT) #endif +// These must be type-generic functions. The C standard specifies them as +// being macros rather than functions, in fact. However, in C++ it's important +// that there be function declarations that don't interfere with other uses of +// the identifier, even in places with parentheses where a function-like macro +// will be expanded (such as a function declaration in a C++ namespace). + +#ifdef __cplusplus + +template inline constexpr bool isfinite(T x) { + return __builtin_isfinite(x); +} + +template inline constexpr bool isinf(T x) { + return __builtin_isinf(x); +} + +template inline constexpr bool isnan(T x) { + return __builtin_isnan(x); +} + +#else + +#define isfinite(x) __builtin_isfinite(x) +#define isinf(x) __builtin_isinf(x) +#define isnan(x) __builtin_isnan(x) + +#endif + #endif // LLVM_LIBC_MACROS_MATH_MACROS_H -- GitLab From 00f3454bbe04ae8cf0eeda981c439e7f97390bd4 Mon Sep 17 00:00:00 2001 From: Slava Zakharin Date: Thu, 21 Mar 2024 15:12:31 -0700 Subject: [PATCH 004/306] [flang][runtime] Added pseudo file unit for simplified PRINT. (#86134) A file unit is emulated via a temporary buffer that accumulates the output, which is printed out via std::printf at the end of the IO statement. This implementation will be used for the offload devices. --- flang/runtime/CMakeLists.txt | 2 + flang/runtime/external-unit.cpp | 333 ++++++++++++++++++++++++++++++++ flang/runtime/io-stmt.cpp | 10 + flang/runtime/lock.h | 17 +- flang/runtime/pseudo-unit.cpp | 167 ++++++++++++++++ flang/runtime/tools.h | 21 ++ flang/runtime/unit.cpp | 319 +----------------------------- flang/runtime/unit.h | 61 +++++- 8 files changed, 615 insertions(+), 315 deletions(-) create mode 100644 flang/runtime/external-unit.cpp create mode 100644 flang/runtime/pseudo-unit.cpp diff --git a/flang/runtime/CMakeLists.txt b/flang/runtime/CMakeLists.txt index 7dd60b5edcd5..021474871154 100644 --- a/flang/runtime/CMakeLists.txt +++ b/flang/runtime/CMakeLists.txt @@ -129,6 +129,7 @@ set(sources exceptions.cpp execute.cpp extensions.cpp + external-unit.cpp extrema.cpp file.cpp findloc.cpp @@ -149,6 +150,7 @@ set(sources numeric.cpp pointer.cpp product.cpp + pseudo-unit.cpp ragged.cpp random.cpp reduction.cpp diff --git a/flang/runtime/external-unit.cpp b/flang/runtime/external-unit.cpp new file mode 100644 index 000000000000..9d650ceca4a8 --- /dev/null +++ b/flang/runtime/external-unit.cpp @@ -0,0 +1,333 @@ +//===-- runtime/external-unit.cpp -----------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Implemenation of ExternalFileUnit for RT_USE_PSEUDO_FILE_UNIT=0. +// +//===----------------------------------------------------------------------===// + +#include "tools.h" + +#if !defined(RT_USE_PSEUDO_FILE_UNIT) + +#include "io-error.h" +#include "lock.h" +#include "unit-map.h" +#include "unit.h" +#include +#include + +namespace Fortran::runtime::io { + +// The per-unit data structures are created on demand so that Fortran I/O +// should work without a Fortran main program. +static Lock unitMapLock; +static Lock createOpenLock; +static UnitMap *unitMap{nullptr}; + +void FlushOutputOnCrash(const Terminator &terminator) { + if (!defaultOutput && !errorOutput) { + return; + } + IoErrorHandler handler{terminator}; + handler.HasIoStat(); // prevent nested crash if flush has error + CriticalSection critical{unitMapLock}; + if (defaultOutput) { + defaultOutput->FlushOutput(handler); + } + if (errorOutput) { + errorOutput->FlushOutput(handler); + } +} + +ExternalFileUnit *ExternalFileUnit::LookUp(int unit) { + return GetUnitMap().LookUp(unit); +} + +ExternalFileUnit *ExternalFileUnit::LookUpOrCreate( + int unit, const Terminator &terminator, bool &wasExtant) { + return GetUnitMap().LookUpOrCreate(unit, terminator, wasExtant); +} + +ExternalFileUnit *ExternalFileUnit::LookUpOrCreateAnonymous(int unit, + Direction dir, Fortran::common::optional isUnformatted, + const Terminator &terminator) { + // Make sure that the returned anonymous unit has been opened + // not just created in the unitMap. + CriticalSection critical{createOpenLock}; + bool exists{false}; + ExternalFileUnit *result{ + GetUnitMap().LookUpOrCreate(unit, terminator, exists)}; + if (result && !exists) { + IoErrorHandler handler{terminator}; + result->OpenAnonymousUnit( + dir == Direction::Input ? OpenStatus::Unknown : OpenStatus::Replace, + Action::ReadWrite, Position::Rewind, Convert::Unknown, handler); + result->isUnformatted = isUnformatted; + } + return result; +} + +ExternalFileUnit *ExternalFileUnit::LookUp( + const char *path, std::size_t pathLen) { + return GetUnitMap().LookUp(path, pathLen); +} + +ExternalFileUnit &ExternalFileUnit::CreateNew( + int unit, const Terminator &terminator) { + bool wasExtant{false}; + ExternalFileUnit *result{ + GetUnitMap().LookUpOrCreate(unit, terminator, wasExtant)}; + RUNTIME_CHECK(terminator, result && !wasExtant); + return *result; +} + +ExternalFileUnit *ExternalFileUnit::LookUpForClose(int unit) { + return GetUnitMap().LookUpForClose(unit); +} + +ExternalFileUnit &ExternalFileUnit::NewUnit( + const Terminator &terminator, bool forChildIo) { + ExternalFileUnit &unit{GetUnitMap().NewUnit(terminator)}; + unit.createdForInternalChildIo_ = forChildIo; + return unit; +} + +bool ExternalFileUnit::OpenUnit(Fortran::common::optional status, + Fortran::common::optional action, Position position, + OwningPtr &&newPath, std::size_t newPathLength, Convert convert, + IoErrorHandler &handler) { + if (convert == Convert::Unknown) { + convert = executionEnvironment.conversion; + } + swapEndianness_ = convert == Convert::Swap || + (convert == Convert::LittleEndian && !isHostLittleEndian) || + (convert == Convert::BigEndian && isHostLittleEndian); + bool impliedClose{false}; + if (IsConnected()) { + bool isSamePath{newPath.get() && path() && pathLength() == newPathLength && + std::memcmp(path(), newPath.get(), newPathLength) == 0}; + if (status && *status != OpenStatus::Old && isSamePath) { + handler.SignalError("OPEN statement for connected unit may not have " + "explicit STATUS= other than 'OLD'"); + return impliedClose; + } + if (!newPath.get() || isSamePath) { + // OPEN of existing unit, STATUS='OLD' or unspecified, not new FILE= + newPath.reset(); + return impliedClose; + } + // Otherwise, OPEN on open unit with new FILE= implies CLOSE + DoImpliedEndfile(handler); + FlushOutput(handler); + TruncateFrame(0, handler); + Close(CloseStatus::Keep, handler); + impliedClose = true; + } + if (newPath.get() && newPathLength > 0) { + if (const auto *already{ + GetUnitMap().LookUp(newPath.get(), newPathLength)}) { + handler.SignalError(IostatOpenAlreadyConnected, + "OPEN(UNIT=%d,FILE='%.*s'): file is already connected to unit %d", + unitNumber_, static_cast(newPathLength), newPath.get(), + already->unitNumber_); + return impliedClose; + } + } + set_path(std::move(newPath), newPathLength); + Open(status.value_or(OpenStatus::Unknown), action, position, handler); + auto totalBytes{knownSize()}; + if (access == Access::Direct) { + if (!openRecl) { + handler.SignalError(IostatOpenBadRecl, + "OPEN(UNIT=%d,ACCESS='DIRECT'): record length is not known", + unitNumber()); + } else if (*openRecl <= 0) { + handler.SignalError(IostatOpenBadRecl, + "OPEN(UNIT=%d,ACCESS='DIRECT',RECL=%jd): record length is invalid", + unitNumber(), static_cast(*openRecl)); + } else if (totalBytes && (*totalBytes % *openRecl != 0)) { + handler.SignalError(IostatOpenBadRecl, + "OPEN(UNIT=%d,ACCESS='DIRECT',RECL=%jd): record length is not an " + "even divisor of the file size %jd", + unitNumber(), static_cast(*openRecl), + static_cast(*totalBytes)); + } + recordLength = openRecl; + } + endfileRecordNumber.reset(); + currentRecordNumber = 1; + if (totalBytes && access == Access::Direct && openRecl.value_or(0) > 0) { + endfileRecordNumber = 1 + (*totalBytes / *openRecl); + } + if (position == Position::Append) { + if (totalBytes) { + frameOffsetInFile_ = *totalBytes; + } + if (access != Access::Stream) { + if (!endfileRecordNumber) { + // Fake it so that we can backspace relative from the end + endfileRecordNumber = std::numeric_limits::max() - 2; + } + currentRecordNumber = *endfileRecordNumber; + } + } + return impliedClose; +} + +void ExternalFileUnit::OpenAnonymousUnit( + Fortran::common::optional status, + Fortran::common::optional action, Position position, + Convert convert, IoErrorHandler &handler) { + // I/O to an unconnected unit reads/creates a local file, e.g. fort.7 + std::size_t pathMaxLen{32}; + auto path{SizedNew{handler}(pathMaxLen)}; + std::snprintf(path.get(), pathMaxLen, "fort.%d", unitNumber_); + OpenUnit(status, action, position, std::move(path), std::strlen(path.get()), + convert, handler); +} + +void ExternalFileUnit::CloseUnit(CloseStatus status, IoErrorHandler &handler) { + DoImpliedEndfile(handler); + FlushOutput(handler); + Close(status, handler); +} + +void ExternalFileUnit::DestroyClosed() { + GetUnitMap().DestroyClosed(*this); // destroys *this +} + +Iostat ExternalFileUnit::SetDirection(Direction direction) { + if (direction == Direction::Input) { + if (mayRead()) { + direction_ = Direction::Input; + return IostatOk; + } else { + return IostatReadFromWriteOnly; + } + } else { + if (mayWrite()) { + direction_ = Direction::Output; + return IostatOk; + } else { + return IostatWriteToReadOnly; + } + } +} + +UnitMap &ExternalFileUnit::CreateUnitMap() { + Terminator terminator{__FILE__, __LINE__}; + IoErrorHandler handler{terminator}; + UnitMap &newUnitMap{*New{terminator}().release()}; + + bool wasExtant{false}; + ExternalFileUnit &out{*newUnitMap.LookUpOrCreate( + FORTRAN_DEFAULT_OUTPUT_UNIT, terminator, wasExtant)}; + RUNTIME_CHECK(terminator, !wasExtant); + out.Predefine(1); + handler.SignalError(out.SetDirection(Direction::Output)); + out.isUnformatted = false; + defaultOutput = &out; + + ExternalFileUnit &in{*newUnitMap.LookUpOrCreate( + FORTRAN_DEFAULT_INPUT_UNIT, terminator, wasExtant)}; + RUNTIME_CHECK(terminator, !wasExtant); + in.Predefine(0); + handler.SignalError(in.SetDirection(Direction::Input)); + in.isUnformatted = false; + defaultInput = ∈ + + ExternalFileUnit &error{ + *newUnitMap.LookUpOrCreate(FORTRAN_ERROR_UNIT, terminator, wasExtant)}; + RUNTIME_CHECK(terminator, !wasExtant); + error.Predefine(2); + handler.SignalError(error.SetDirection(Direction::Output)); + error.isUnformatted = false; + errorOutput = &error; + + return newUnitMap; +} + +// A back-up atexit() handler for programs that don't terminate with a main +// program END or a STOP statement or other Fortran-initiated program shutdown, +// such as programs with a C main() that terminate normally. It flushes all +// external I/O units. It is registered once the first time that any external +// I/O is attempted. +static void CloseAllExternalUnits() { + IoErrorHandler handler{"Fortran program termination"}; + ExternalFileUnit::CloseAll(handler); +} + +UnitMap &ExternalFileUnit::GetUnitMap() { + if (unitMap) { + return *unitMap; + } + { + CriticalSection critical{unitMapLock}; + if (unitMap) { + return *unitMap; + } + unitMap = &CreateUnitMap(); + } + std::atexit(CloseAllExternalUnits); + return *unitMap; +} + +void ExternalFileUnit::CloseAll(IoErrorHandler &handler) { + CriticalSection critical{unitMapLock}; + if (unitMap) { + unitMap->CloseAll(handler); + FreeMemoryAndNullify(unitMap); + } + defaultOutput = nullptr; + defaultInput = nullptr; + errorOutput = nullptr; +} + +void ExternalFileUnit::FlushAll(IoErrorHandler &handler) { + CriticalSection critical{unitMapLock}; + if (unitMap) { + unitMap->FlushAll(handler); + } +} + +int ExternalFileUnit::GetAsynchronousId(IoErrorHandler &handler) { + if (!mayAsynchronous()) { + handler.SignalError(IostatBadAsynchronous); + return -1; + } else { + for (int j{0}; 64 * j < maxAsyncIds; ++j) { + if (auto least{asyncIdAvailable_[j].LeastElement()}) { + asyncIdAvailable_[j].reset(*least); + return 64 * j + static_cast(*least); + } + } + handler.SignalError(IostatTooManyAsyncOps); + return -1; + } +} + +bool ExternalFileUnit::Wait(int id) { + if (static_cast(id) >= maxAsyncIds || + asyncIdAvailable_[id / 64].test(id % 64)) { + return false; + } else { + if (id == 0) { // means "all IDs" + for (int j{0}; 64 * j < maxAsyncIds; ++j) { + asyncIdAvailable_[j].set(); + } + asyncIdAvailable_[0].reset(0); + } else { + asyncIdAvailable_[id / 64].set(id % 64); + } + return true; + } +} + +} // namespace Fortran::runtime::io + +#endif // !defined(RT_USE_PSEUDO_FILE_UNIT) diff --git a/flang/runtime/io-stmt.cpp b/flang/runtime/io-stmt.cpp index 075d7b5ae518..e3f1214324d8 100644 --- a/flang/runtime/io-stmt.cpp +++ b/flang/runtime/io-stmt.cpp @@ -227,7 +227,17 @@ ConnectionState &ExternalIoStatementBase::GetConnectionState() { return unit_; } int ExternalIoStatementBase::EndIoStatement() { CompleteOperation(); auto result{IoStatementBase::EndIoStatement()}; +#if !defined(RT_USE_PSEUDO_FILE_UNIT) unit_.EndIoStatement(); // annihilates *this in unit_.u_ +#else + // Fetch the unit pointer before *this disappears. + ExternalFileUnit *unitPtr{&unit_}; + // The pseudo file units are dynamically allocated + // and are not tracked in the unit map. + // They have to be destructed and deallocated here. + unitPtr->~ExternalFileUnit(); + FreeMemory(unitPtr); +#endif return result; } diff --git a/flang/runtime/lock.h b/flang/runtime/lock.h index 5fdcf4745c21..61b06a62ff7c 100644 --- a/flang/runtime/lock.h +++ b/flang/runtime/lock.h @@ -12,6 +12,7 @@ #define FORTRAN_RUNTIME_LOCK_H_ #include "terminator.h" +#include "tools.h" // Avoid if possible to avoid introduction of C++ runtime // library dependence. @@ -35,7 +36,17 @@ namespace Fortran::runtime { class Lock { public: -#if USE_PTHREADS +#if RT_USE_PSEUDO_LOCK + // No lock implementation, e.g. for using together + // with RT_USE_PSEUDO_FILE_UNIT. + // The users of Lock class may use it under + // USE_PTHREADS and otherwise, so it has to provide + // all the interfaces. + void Take() {} + bool Try() { return true; } + void Drop() {} + bool TakeIfNoDeadlock() { return true; } +#elif USE_PTHREADS Lock() { pthread_mutex_init(&mutex_, nullptr); } ~Lock() { pthread_mutex_destroy(&mutex_); } void Take() { @@ -79,7 +90,9 @@ public: } private: -#if USE_PTHREADS +#if RT_USE_PSEUDO_FILE_UNIT + // No state. +#elif USE_PTHREADS pthread_mutex_t mutex_{}; volatile bool isBusy_{false}; volatile pthread_t holder_; diff --git a/flang/runtime/pseudo-unit.cpp b/flang/runtime/pseudo-unit.cpp new file mode 100644 index 000000000000..8b5f36e2233a --- /dev/null +++ b/flang/runtime/pseudo-unit.cpp @@ -0,0 +1,167 @@ +//===-- runtime/pseudo-unit.cpp -------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Implemenation of ExternalFileUnit and PseudoOpenFile for +// RT_USE_PSEUDO_FILE_UNIT=1. +// +//===----------------------------------------------------------------------===// + +#include "tools.h" + +#if defined(RT_USE_PSEUDO_FILE_UNIT) + +#include "io-error.h" +#include "unit.h" +#include + +namespace Fortran::runtime::io { + +void FlushOutputOnCrash(const Terminator &) {} + +ExternalFileUnit *ExternalFileUnit::LookUp(int) { + Terminator{__FILE__, __LINE__}.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +ExternalFileUnit *ExternalFileUnit::LookUpOrCreate( + int, const Terminator &, bool &) { + Terminator{__FILE__, __LINE__}.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +ExternalFileUnit *ExternalFileUnit::LookUpOrCreateAnonymous(int unit, + Direction direction, Fortran::common::optional, + const Terminator &terminator) { + if (direction != Direction::Output) { + terminator.Crash("ExternalFileUnit only supports output IO"); + } + return New{terminator}(unit).release(); +} + +ExternalFileUnit *ExternalFileUnit::LookUp(const char *, std::size_t) { + Terminator{__FILE__, __LINE__}.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +ExternalFileUnit &ExternalFileUnit::CreateNew(int, const Terminator &) { + Terminator{__FILE__, __LINE__}.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +ExternalFileUnit *ExternalFileUnit::LookUpForClose(int) { + Terminator{__FILE__, __LINE__}.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +ExternalFileUnit &ExternalFileUnit::NewUnit(const Terminator &, bool) { + Terminator{__FILE__, __LINE__}.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +bool ExternalFileUnit::OpenUnit(Fortran::common::optional status, + Fortran::common::optional, Position, OwningPtr &&, + std::size_t, Convert, IoErrorHandler &handler) { + handler.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +void ExternalFileUnit::OpenAnonymousUnit(Fortran::common::optional, + Fortran::common::optional, Position, Convert convert, + IoErrorHandler &handler) { + handler.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +void ExternalFileUnit::CloseUnit(CloseStatus, IoErrorHandler &handler) { + handler.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +void ExternalFileUnit::DestroyClosed() { + Terminator{__FILE__, __LINE__}.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +Iostat ExternalFileUnit::SetDirection(Direction direction) { + if (direction != Direction::Output) { + return IostatReadFromWriteOnly; + } + direction_ = direction; + return IostatOk; +} + +void ExternalFileUnit::CloseAll(IoErrorHandler &) {} + +void ExternalFileUnit::FlushAll(IoErrorHandler &) {} + +int ExternalFileUnit::GetAsynchronousId(IoErrorHandler &handler) { + handler.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +bool ExternalFileUnit::Wait(int) { + Terminator{__FILE__, __LINE__}.Crash("unsupported"); +} + +void PseudoOpenFile::set_mayAsynchronous(bool yes) { + if (yes) { + Terminator{__FILE__, __LINE__}.Crash("%s: unsupported", RT_PRETTY_FUNCTION); + } +} + +Fortran::common::optional +PseudoOpenFile::knownSize() const { + Terminator{__FILE__, __LINE__}.Crash("unsupported"); +} + +void PseudoOpenFile::Open(OpenStatus, Fortran::common::optional, + Position, IoErrorHandler &handler) { + handler.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +void PseudoOpenFile::Close(CloseStatus, IoErrorHandler &handler) { + handler.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +std::size_t PseudoOpenFile::Read( + FileOffset, char *, std::size_t, std::size_t, IoErrorHandler &handler) { + handler.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +std::size_t PseudoOpenFile::Write(FileOffset at, const char *buffer, + std::size_t bytes, IoErrorHandler &handler) { + if (at) { + handler.Crash("%s: unsupported", RT_PRETTY_FUNCTION); + } + // TODO: use persistent string buffer that can be reallocated + // as needed, and only freed at destruction of *this. + auto string{SizedNew{handler}(bytes + 1)}; + std::memcpy(string.get(), buffer, bytes); + string.get()[bytes] = '\0'; + std::printf("%s", string.get()); + return bytes; +} + +void PseudoOpenFile::Truncate(FileOffset, IoErrorHandler &handler) { + handler.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +int PseudoOpenFile::ReadAsynchronously( + FileOffset, char *, std::size_t, IoErrorHandler &handler) { + handler.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +int PseudoOpenFile::WriteAsynchronously( + FileOffset, const char *, std::size_t, IoErrorHandler &handler) { + handler.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +void PseudoOpenFile::Wait(int, IoErrorHandler &handler) { + handler.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +void PseudoOpenFile::WaitAll(IoErrorHandler &handler) { + handler.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +Position PseudoOpenFile::InquirePosition() const { + Terminator{__FILE__, __LINE__}.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +} // namespace Fortran::runtime::io + +#endif // defined(RT_USE_PSEUDO_FILE_UNIT) diff --git a/flang/runtime/tools.h b/flang/runtime/tools.h index df25eb888233..c70a1b438e33 100644 --- a/flang/runtime/tools.h +++ b/flang/runtime/tools.h @@ -21,6 +21,27 @@ #include #include +/// \macro RT_PRETTY_FUNCTION +/// Gets a user-friendly looking function signature for the current scope +/// using the best available method on each platform. The exact format of the +/// resulting string is implementation specific and non-portable, so this should +/// only be used, for example, for logging or diagnostics. +/// Copy of LLVM_PRETTY_FUNCTION +#if defined(_MSC_VER) +#define RT_PRETTY_FUNCTION __FUNCSIG__ +#elif defined(__GNUC__) || defined(__clang__) +#define RT_PRETTY_FUNCTION __PRETTY_FUNCTION__ +#else +#define RT_PRETTY_FUNCTION __func__ +#endif + +#if defined(RT_DEVICE_COMPILATION) +// Use the pseudo lock and pseudo file unit implementations +// for the device. +#define RT_USE_PSEUDO_LOCK 1 +#define RT_USE_PSEUDO_FILE_UNIT 1 +#endif + namespace Fortran::runtime { class Terminator; diff --git a/flang/runtime/unit.cpp b/flang/runtime/unit.cpp index 82f0e68cc20a..67f4775ae0a9 100644 --- a/flang/runtime/unit.cpp +++ b/flang/runtime/unit.cpp @@ -5,293 +5,23 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// - +// +// Implementation of ExternalFileUnit common for both +// RT_USE_PSEUDO_FILE_UNIT=0 and RT_USE_PSEUDO_FILE_UNIT=1. +// +//===----------------------------------------------------------------------===// #include "unit.h" #include "io-error.h" #include "lock.h" #include "tools.h" -#include "unit-map.h" -#include "flang/Runtime/magic-numbers.h" -#include #include #include namespace Fortran::runtime::io { -// The per-unit data structures are created on demand so that Fortran I/O -// should work without a Fortran main program. -static Lock unitMapLock; -static Lock createOpenLock; -static UnitMap *unitMap{nullptr}; -static ExternalFileUnit *defaultInput{nullptr}; // unit 5 -static ExternalFileUnit *defaultOutput{nullptr}; // unit 6 -static ExternalFileUnit *errorOutput{nullptr}; // unit 0 extension - -void FlushOutputOnCrash(const Terminator &terminator) { - if (!defaultOutput && !errorOutput) { - return; - } - IoErrorHandler handler{terminator}; - handler.HasIoStat(); // prevent nested crash if flush has error - CriticalSection critical{unitMapLock}; - if (defaultOutput) { - defaultOutput->FlushOutput(handler); - } - if (errorOutput) { - errorOutput->FlushOutput(handler); - } -} - -ExternalFileUnit *ExternalFileUnit::LookUp(int unit) { - return GetUnitMap().LookUp(unit); -} - -ExternalFileUnit *ExternalFileUnit::LookUpOrCreate( - int unit, const Terminator &terminator, bool &wasExtant) { - return GetUnitMap().LookUpOrCreate(unit, terminator, wasExtant); -} - -ExternalFileUnit *ExternalFileUnit::LookUpOrCreateAnonymous(int unit, - Direction dir, Fortran::common::optional isUnformatted, - const Terminator &terminator) { - // Make sure that the returned anonymous unit has been opened - // not just created in the unitMap. - CriticalSection critical{createOpenLock}; - bool exists{false}; - ExternalFileUnit *result{ - GetUnitMap().LookUpOrCreate(unit, terminator, exists)}; - if (result && !exists) { - IoErrorHandler handler{terminator}; - result->OpenAnonymousUnit( - dir == Direction::Input ? OpenStatus::Unknown : OpenStatus::Replace, - Action::ReadWrite, Position::Rewind, Convert::Unknown, handler); - result->isUnformatted = isUnformatted; - } - return result; -} - -ExternalFileUnit *ExternalFileUnit::LookUp( - const char *path, std::size_t pathLen) { - return GetUnitMap().LookUp(path, pathLen); -} - -ExternalFileUnit &ExternalFileUnit::CreateNew( - int unit, const Terminator &terminator) { - bool wasExtant{false}; - ExternalFileUnit *result{ - GetUnitMap().LookUpOrCreate(unit, terminator, wasExtant)}; - RUNTIME_CHECK(terminator, result && !wasExtant); - return *result; -} - -ExternalFileUnit *ExternalFileUnit::LookUpForClose(int unit) { - return GetUnitMap().LookUpForClose(unit); -} - -ExternalFileUnit &ExternalFileUnit::NewUnit( - const Terminator &terminator, bool forChildIo) { - ExternalFileUnit &unit{GetUnitMap().NewUnit(terminator)}; - unit.createdForInternalChildIo_ = forChildIo; - return unit; -} - -bool ExternalFileUnit::OpenUnit(Fortran::common::optional status, - Fortran::common::optional action, Position position, - OwningPtr &&newPath, std::size_t newPathLength, Convert convert, - IoErrorHandler &handler) { - if (convert == Convert::Unknown) { - convert = executionEnvironment.conversion; - } - swapEndianness_ = convert == Convert::Swap || - (convert == Convert::LittleEndian && !isHostLittleEndian) || - (convert == Convert::BigEndian && isHostLittleEndian); - bool impliedClose{false}; - if (IsConnected()) { - bool isSamePath{newPath.get() && path() && pathLength() == newPathLength && - std::memcmp(path(), newPath.get(), newPathLength) == 0}; - if (status && *status != OpenStatus::Old && isSamePath) { - handler.SignalError("OPEN statement for connected unit may not have " - "explicit STATUS= other than 'OLD'"); - return impliedClose; - } - if (!newPath.get() || isSamePath) { - // OPEN of existing unit, STATUS='OLD' or unspecified, not new FILE= - newPath.reset(); - return impliedClose; - } - // Otherwise, OPEN on open unit with new FILE= implies CLOSE - DoImpliedEndfile(handler); - FlushOutput(handler); - TruncateFrame(0, handler); - Close(CloseStatus::Keep, handler); - impliedClose = true; - } - if (newPath.get() && newPathLength > 0) { - if (const auto *already{ - GetUnitMap().LookUp(newPath.get(), newPathLength)}) { - handler.SignalError(IostatOpenAlreadyConnected, - "OPEN(UNIT=%d,FILE='%.*s'): file is already connected to unit %d", - unitNumber_, static_cast(newPathLength), newPath.get(), - already->unitNumber_); - return impliedClose; - } - } - set_path(std::move(newPath), newPathLength); - Open(status.value_or(OpenStatus::Unknown), action, position, handler); - auto totalBytes{knownSize()}; - if (access == Access::Direct) { - if (!openRecl) { - handler.SignalError(IostatOpenBadRecl, - "OPEN(UNIT=%d,ACCESS='DIRECT'): record length is not known", - unitNumber()); - } else if (*openRecl <= 0) { - handler.SignalError(IostatOpenBadRecl, - "OPEN(UNIT=%d,ACCESS='DIRECT',RECL=%jd): record length is invalid", - unitNumber(), static_cast(*openRecl)); - } else if (totalBytes && (*totalBytes % *openRecl != 0)) { - handler.SignalError(IostatOpenBadRecl, - "OPEN(UNIT=%d,ACCESS='DIRECT',RECL=%jd): record length is not an " - "even divisor of the file size %jd", - unitNumber(), static_cast(*openRecl), - static_cast(*totalBytes)); - } - recordLength = openRecl; - } - endfileRecordNumber.reset(); - currentRecordNumber = 1; - if (totalBytes && access == Access::Direct && openRecl.value_or(0) > 0) { - endfileRecordNumber = 1 + (*totalBytes / *openRecl); - } - if (position == Position::Append) { - if (totalBytes) { - frameOffsetInFile_ = *totalBytes; - } - if (access != Access::Stream) { - if (!endfileRecordNumber) { - // Fake it so that we can backspace relative from the end - endfileRecordNumber = std::numeric_limits::max() - 2; - } - currentRecordNumber = *endfileRecordNumber; - } - } - return impliedClose; -} - -void ExternalFileUnit::OpenAnonymousUnit( - Fortran::common::optional status, - Fortran::common::optional action, Position position, - Convert convert, IoErrorHandler &handler) { - // I/O to an unconnected unit reads/creates a local file, e.g. fort.7 - std::size_t pathMaxLen{32}; - auto path{SizedNew{handler}(pathMaxLen)}; - std::snprintf(path.get(), pathMaxLen, "fort.%d", unitNumber_); - OpenUnit(status, action, position, std::move(path), std::strlen(path.get()), - convert, handler); -} - -void ExternalFileUnit::CloseUnit(CloseStatus status, IoErrorHandler &handler) { - DoImpliedEndfile(handler); - FlushOutput(handler); - Close(status, handler); -} - -void ExternalFileUnit::DestroyClosed() { - GetUnitMap().DestroyClosed(*this); // destroys *this -} - -Iostat ExternalFileUnit::SetDirection(Direction direction) { - if (direction == Direction::Input) { - if (mayRead()) { - direction_ = Direction::Input; - return IostatOk; - } else { - return IostatReadFromWriteOnly; - } - } else { - if (mayWrite()) { - direction_ = Direction::Output; - return IostatOk; - } else { - return IostatWriteToReadOnly; - } - } -} - -UnitMap &ExternalFileUnit::CreateUnitMap() { - Terminator terminator{__FILE__, __LINE__}; - IoErrorHandler handler{terminator}; - UnitMap &newUnitMap{*New{terminator}().release()}; - - bool wasExtant{false}; - ExternalFileUnit &out{*newUnitMap.LookUpOrCreate( - FORTRAN_DEFAULT_OUTPUT_UNIT, terminator, wasExtant)}; - RUNTIME_CHECK(terminator, !wasExtant); - out.Predefine(1); - handler.SignalError(out.SetDirection(Direction::Output)); - out.isUnformatted = false; - defaultOutput = &out; - - ExternalFileUnit &in{*newUnitMap.LookUpOrCreate( - FORTRAN_DEFAULT_INPUT_UNIT, terminator, wasExtant)}; - RUNTIME_CHECK(terminator, !wasExtant); - in.Predefine(0); - handler.SignalError(in.SetDirection(Direction::Input)); - in.isUnformatted = false; - defaultInput = ∈ - - ExternalFileUnit &error{ - *newUnitMap.LookUpOrCreate(FORTRAN_ERROR_UNIT, terminator, wasExtant)}; - RUNTIME_CHECK(terminator, !wasExtant); - error.Predefine(2); - handler.SignalError(error.SetDirection(Direction::Output)); - error.isUnformatted = false; - errorOutput = &error; - - return newUnitMap; -} - -// A back-up atexit() handler for programs that don't terminate with a main -// program END or a STOP statement or other Fortran-initiated program shutdown, -// such as programs with a C main() that terminate normally. It flushes all -// external I/O units. It is registered once the first time that any external -// I/O is attempted. -static void CloseAllExternalUnits() { - IoErrorHandler handler{"Fortran program termination"}; - ExternalFileUnit::CloseAll(handler); -} - -UnitMap &ExternalFileUnit::GetUnitMap() { - if (unitMap) { - return *unitMap; - } - { - CriticalSection critical{unitMapLock}; - if (unitMap) { - return *unitMap; - } - unitMap = &CreateUnitMap(); - } - std::atexit(CloseAllExternalUnits); - return *unitMap; -} - -void ExternalFileUnit::CloseAll(IoErrorHandler &handler) { - CriticalSection critical{unitMapLock}; - if (unitMap) { - unitMap->CloseAll(handler); - FreeMemoryAndNullify(unitMap); - } - defaultOutput = nullptr; - defaultInput = nullptr; - errorOutput = nullptr; -} - -void ExternalFileUnit::FlushAll(IoErrorHandler &handler) { - CriticalSection critical{unitMapLock}; - if (unitMap) { - unitMap->FlushAll(handler); - } -} +ExternalFileUnit *defaultInput{nullptr}; // unit 5 +ExternalFileUnit *defaultOutput{nullptr}; // unit 6 +ExternalFileUnit *errorOutput{nullptr}; // unit 0 extension static inline void SwapEndianness( char *data, std::size_t bytes, std::size_t elementBytes) { @@ -999,39 +729,6 @@ void ExternalFileUnit::PopChildIo(ChildIo &child) { child_.reset(child.AcquirePrevious().release()); // deletes top child } -int ExternalFileUnit::GetAsynchronousId(IoErrorHandler &handler) { - if (!mayAsynchronous()) { - handler.SignalError(IostatBadAsynchronous); - return -1; - } else { - for (int j{0}; 64 * j < maxAsyncIds; ++j) { - if (auto least{asyncIdAvailable_[j].LeastElement()}) { - asyncIdAvailable_[j].reset(*least); - return 64 * j + static_cast(*least); - } - } - handler.SignalError(IostatTooManyAsyncOps); - return -1; - } -} - -bool ExternalFileUnit::Wait(int id) { - if (static_cast(id) >= maxAsyncIds || - asyncIdAvailable_[id / 64].test(id % 64)) { - return false; - } else { - if (id == 0) { // means "all IDs" - for (int j{0}; 64 * j < maxAsyncIds; ++j) { - asyncIdAvailable_[j].set(); - } - asyncIdAvailable_[0].reset(0); - } else { - asyncIdAvailable_[id / 64].set(id % 64); - } - return true; - } -} - std::int32_t ExternalFileUnit::ReadHeaderOrFooter(std::int64_t frameOffset) { std::int32_t word; char *wordPtr{reinterpret_cast(&word)}; diff --git a/flang/runtime/unit.h b/flang/runtime/unit.h index fc5bead7e1d9..5f854abd42f6 100644 --- a/flang/runtime/unit.h +++ b/flang/runtime/unit.h @@ -31,10 +31,67 @@ namespace Fortran::runtime::io { class UnitMap; class ChildIo; +class ExternalFileUnit; + +// Predefined file units. +extern ExternalFileUnit *defaultInput; // unit 5 +extern ExternalFileUnit *defaultOutput; // unit 6 +extern ExternalFileUnit *errorOutput; // unit 0 extension + +#if defined(RT_USE_PSEUDO_FILE_UNIT) +// A flavor of OpenFile class that pretends to be a terminal, +// and only provides basic buffering of the output +// in an internal buffer, and Write's the output +// using std::printf(). Since it does not rely on file system +// APIs, it can be used to implement external output +// for offload devices. +class PseudoOpenFile { +public: + using FileOffset = std::int64_t; + + const char *path() const { return nullptr; } + std::size_t pathLength() const { return 0; } + void set_path(OwningPtr &&, std::size_t bytes) {} + bool mayRead() const { return false; } + bool mayWrite() const { return true; } + bool mayPosition() const { return false; } + bool mayAsynchronous() const { return false; } + void set_mayAsynchronous(bool yes); + // Pretend to be a terminal to force the output + // at the end of IO statement. + bool isTerminal() const { return true; } + bool isWindowsTextFile() const { return false; } + Fortran::common::optional knownSize() const; + bool IsConnected() const { return false; } + void Open(OpenStatus, Fortran::common::optional, Position, + IoErrorHandler &); + void Predefine(int fd) {} + void Close(CloseStatus, IoErrorHandler &); + std::size_t Read(FileOffset, char *, std::size_t minBytes, + std::size_t maxBytes, IoErrorHandler &); + std::size_t Write(FileOffset, const char *, std::size_t, IoErrorHandler &); + void Truncate(FileOffset, IoErrorHandler &); + int ReadAsynchronously(FileOffset, char *, std::size_t, IoErrorHandler &); + int WriteAsynchronously( + FileOffset, const char *, std::size_t, IoErrorHandler &); + void Wait(int id, IoErrorHandler &); + void WaitAll(IoErrorHandler &); + Position InquirePosition() const; +}; +#endif // defined(RT_USE_PSEUDO_FILE_UNIT) + +#if !defined(RT_USE_PSEUDO_FILE_UNIT) +using OpenFileClass = OpenFile; +using FileFrameClass = FileFrame; +#else // defined(RT_USE_PSEUDO_FILE_UNIT) +using OpenFileClass = PseudoOpenFile; +// Use not so big buffer for the pseudo file unit frame. +using FileFrameClass = FileFrame; +#endif // defined(RT_USE_PSEUDO_FILE_UNIT) class ExternalFileUnit : public ConnectionState, - public OpenFile, - public FileFrame { + public OpenFileClass, + public FileFrameClass { public: static constexpr int maxAsyncIds{64 * 16}; -- GitLab From 6f9297fc4da9df776aef7ee9a18ac426053aaed4 Mon Sep 17 00:00:00 2001 From: Alexander Richardson Date: Thu, 21 Mar 2024 15:22:06 -0700 Subject: [PATCH 005/306] [compiler-rt] Fix build race with COMPILER_RT_TEST_STANDALONE_BUILD_LIBS Since this standalone build configuration uses the runtime libraries that are being built just now, we need to ensure that e.g. the TSan unit tests depend on the tsan runtime library. Also fix TSAN_DEPS being overridden to not include the tsan runtime (commit .....). This change fixes a build race seen in the CI checks for TsanRtlTest-x86_64-Test in https://github.com/llvm/llvm-project/pull/83088. Reviewed By: vitalybuka Pull Request: https://github.com/llvm/llvm-project/pull/83650 --- compiler-rt/cmake/Modules/CompilerRTCompile.cmake | 2 +- compiler-rt/lib/msan/tests/CMakeLists.txt | 6 +++--- compiler-rt/lib/tsan/tests/CMakeLists.txt | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/compiler-rt/cmake/Modules/CompilerRTCompile.cmake b/compiler-rt/cmake/Modules/CompilerRTCompile.cmake index d5b2e7970f11..1629db18f1c2 100644 --- a/compiler-rt/cmake/Modules/CompilerRTCompile.cmake +++ b/compiler-rt/cmake/Modules/CompilerRTCompile.cmake @@ -46,7 +46,7 @@ function(sanitizer_test_compile obj_list source arch) # Write out architecture-specific flags into TARGET_CFLAGS variable. get_target_flags_for_arch(${arch} TARGET_CFLAGS) set(COMPILE_DEPS ${TEST_COMPILE_DEPS}) - if(NOT COMPILER_RT_STANDALONE_BUILD) + if(NOT COMPILER_RT_STANDALONE_BUILD OR COMPILER_RT_TEST_STANDALONE_BUILD_LIBS) list(APPEND COMPILE_DEPS ${TEST_DEPS}) endif() clang_compile(${output_obj} ${source} diff --git a/compiler-rt/lib/msan/tests/CMakeLists.txt b/compiler-rt/lib/msan/tests/CMakeLists.txt index 4f09f1e6a691..e0771dd5a1a7 100644 --- a/compiler-rt/lib/msan/tests/CMakeLists.txt +++ b/compiler-rt/lib/msan/tests/CMakeLists.txt @@ -69,9 +69,9 @@ macro(msan_compile obj_list source arch kind cflags) sanitizer_test_compile( ${obj_list} ${source} ${arch} KIND ${kind} - COMPILE_DEPS ${MSAN_UNITTEST_HEADERS} + COMPILE_DEPS ${MSAN_UNITTEST_HEADERS} libcxx_msan_${arch}-build DEPS msan - CFLAGS -isystem ${CMAKE_CURRENT_BINARY_DIR}/../libcxx_msan_${arch}/include/c++/v1 + CFLAGS -isystem ${MSAN_LIBCXX_DIR}/../include/c++/v1 ${MSAN_UNITTEST_INSTRUMENTED_CFLAGS} ${cflags} ) endmacro() @@ -120,7 +120,7 @@ macro(add_msan_tests_for_arch arch kind cflags) set(MSAN_TEST_DEPS ${MSAN_TEST_OBJECTS} libcxx_msan_${arch}-build ${MSAN_LOADABLE_SO} "${MSAN_LIBCXX_DIR}/libc++.a" "${MSAN_LIBCXX_DIR}/libc++abi.a") - list(APPEND MSAN_TEST_DEPS msan) + list(APPEND MSAN_TEST_DEPS msan libcxx_msan_${arch}-build) get_target_flags_for_arch(${arch} TARGET_LINK_FLAGS) add_compiler_rt_test(MsanUnitTests "Msan-${arch}${kind}-Test" ${arch} OBJECTS ${MSAN_TEST_OBJECTS} "${MSAN_LIBCXX_DIR}/libc++.a" "${MSAN_LIBCXX_DIR}/libc++abi.a" diff --git a/compiler-rt/lib/tsan/tests/CMakeLists.txt b/compiler-rt/lib/tsan/tests/CMakeLists.txt index ad8cc9b0eb05..1bc08bbf7450 100644 --- a/compiler-rt/lib/tsan/tests/CMakeLists.txt +++ b/compiler-rt/lib/tsan/tests/CMakeLists.txt @@ -67,7 +67,7 @@ endforeach() set(TSAN_DEPS tsan) # TSan uses C++ standard library headers. if (TARGET cxx-headers OR HAVE_LIBCXX) - set(TSAN_DEPS cxx-headers) + list(APPEND TSAN_DEPS cxx-headers) endif() # add_tsan_unittest( -- GitLab From 8d1affb87181b9636b87e04a245bcde06f8a7d47 Mon Sep 17 00:00:00 2001 From: Diego Caballero Date: Thu, 21 Mar 2024 15:28:36 -0700 Subject: [PATCH 006/306] Update @dcaballe in CODEOWNERS (#86177) It fixes a few rules that don't seem to be working and adding myself to a few paths where I've been contributing and can offer my review. Also minor sorting changes. --- .github/CODEOWNERS | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index c246c42b0904..9cf6f5d79d41 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -59,8 +59,8 @@ clang/test/AST/Interp/ @tbaederr /mlir/Dialect/*/Transforms/Bufferize.cpp @matthias-springer # Linalg Dialect in MLIR. -/mlir/include/mlir/Dialect/Linalg @dcaballe @nicolasvasilache -/mlir/lib/Dialect/Linalg @dcaballe @nicolasvasilache +/mlir/include/mlir/Dialect/Linalg/* @dcaballe @nicolasvasilache +/mlir/lib/Dialect/Linalg/* @dcaballe @nicolasvasilache /mlir/lib/Dialect/Linalg/Transforms/DecomposeLinalgOps.cpp @MaheshRavishankar @nicolasvasilache /mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp @MaheshRavishankar @nicolasvasilache /mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp @MaheshRavishankar @nicolasvasilache @@ -77,14 +77,14 @@ clang/test/AST/Interp/ @tbaederr /mlir/**/*SME* @banach-space @dcaballe @nicolasvasilache /mlir/**/*SVE* @banach-space @dcaballe @nicolasvasilache /mlir/**/*VectorInterfaces* @dcaballe @nicolasvasilache -/mlir/**/*VectorToSCF* @banach-space @dcaballe @nicolasvasilache @matthias-springer +/mlir/**/*VectorToSCF* @banach-space @dcaballe @matthias-springer @nicolasvasilache /mlir/**/*VectorToLLVM* @banach-space @dcaballe @nicolasvasilache /mlir/**/*X86Vector* @aartbik @dcaballe @nicolasvasilache -/mlir/include/mlir/Dialect/Vector @dcaballe @nicolasvasilache -/mlir/lib/Dialect/Vector @dcaballe @nicolasvasilache -/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp @MaheshRavishankar @nicolasvasilache -/mlir/**/*EmulateNarrowType* @hanhanW +/mlir/include/mlir/Dialect/Vector/* @dcaballe @nicolasvasilache +/mlir/lib/Dialect/Vector/* @dcaballe @nicolasvasilache /mlir/lib/Dialect/Vector/Transforms/* @hanhanW @nicolasvasilache +/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp @MaheshRavishankar @nicolasvasilache +/mlir/**/*EmulateNarrowType* @dcaballe @hanhanW # Presburger library in MLIR /mlir/**/*Presburger* @Groverkss @Superty -- GitLab From 1538b82fd395a2fba90842b8a8010b8dcc919499 Mon Sep 17 00:00:00 2001 From: Cooper Partin Date: Thu, 21 Mar 2024 15:40:29 -0700 Subject: [PATCH 007/306] Revert "Add support for PSV EntryFunctionName (#84409)" (#86211) This reverts commit cde54df39cab3a1d60a3e1862ab341609bee3cc3. Co-authored-by: Cooper Partin --- llvm/include/llvm/BinaryFormat/DXContainer.h | 13 --- llvm/include/llvm/MC/DXContainerPSVInfo.h | 25 ++-- llvm/include/llvm/MC/StringTableBuilder.h | 8 +- llvm/include/llvm/Object/DXContainer.h | 16 +-- .../include/llvm/ObjectYAML/DXContainerYAML.h | 5 +- llvm/lib/MC/DXContainerPSVInfo.cpp | 75 ++++-------- llvm/lib/Object/DXContainer.cpp | 15 +-- llvm/lib/ObjectYAML/DXContainerEmitter.cpp | 3 +- llvm/lib/ObjectYAML/DXContainerYAML.cpp | 15 --- .../DXContainer/PSVv3-amplification.yaml | 97 ---------------- .../ObjectYAML/DXContainer/PSVv3-compute.yaml | 95 --------------- .../ObjectYAML/DXContainer/PSVv3-domain.yaml | 105 ----------------- .../DXContainer/PSVv3-geometry.yaml | 105 ----------------- .../ObjectYAML/DXContainer/PSVv3-hull.yaml | 107 ----------------- .../ObjectYAML/DXContainer/PSVv3-mesh.yaml | 109 ------------------ .../ObjectYAML/DXContainer/PSVv3-pixel.yaml | 99 ---------------- .../ObjectYAML/DXContainer/PSVv3-vertex.yaml | 97 ---------------- llvm/tools/obj2yaml/dxcontainer2yaml.cpp | 3 - 18 files changed, 52 insertions(+), 940 deletions(-) delete mode 100644 llvm/test/ObjectYAML/DXContainer/PSVv3-amplification.yaml delete mode 100644 llvm/test/ObjectYAML/DXContainer/PSVv3-compute.yaml delete mode 100644 llvm/test/ObjectYAML/DXContainer/PSVv3-domain.yaml delete mode 100644 llvm/test/ObjectYAML/DXContainer/PSVv3-geometry.yaml delete mode 100644 llvm/test/ObjectYAML/DXContainer/PSVv3-hull.yaml delete mode 100644 llvm/test/ObjectYAML/DXContainer/PSVv3-mesh.yaml delete mode 100644 llvm/test/ObjectYAML/DXContainer/PSVv3-pixel.yaml delete mode 100644 llvm/test/ObjectYAML/DXContainer/PSVv3-vertex.yaml diff --git a/llvm/include/llvm/BinaryFormat/DXContainer.h b/llvm/include/llvm/BinaryFormat/DXContainer.h index ba882c4a6f32..532f9481766a 100644 --- a/llvm/include/llvm/BinaryFormat/DXContainer.h +++ b/llvm/include/llvm/BinaryFormat/DXContainer.h @@ -424,19 +424,6 @@ struct ResourceBindInfo : public v0::ResourceBindInfo { }; } // namespace v2 - -namespace v3 { -struct RuntimeInfo : public v2::RuntimeInfo { - uint32_t EntryNameOffset; - - void swapBytes() { sys::swapByteOrder(EntryNameOffset); } - - void swapBytes(Triple::EnvironmentType Stage) { - v2::RuntimeInfo::swapBytes(Stage); - } -}; - -} // namespace v3 } // namespace PSV #define COMPONENT_PRECISION(Val, Enum) Enum = Val, diff --git a/llvm/include/llvm/MC/DXContainerPSVInfo.h b/llvm/include/llvm/MC/DXContainerPSVInfo.h index bad2fe78eb8f..7d21c18d252f 100644 --- a/llvm/include/llvm/MC/DXContainerPSVInfo.h +++ b/llvm/include/llvm/MC/DXContainerPSVInfo.h @@ -9,11 +9,9 @@ #ifndef LLVM_MC_DXCONTAINERPSVINFO_H #define LLVM_MC_DXCONTAINERPSVINFO_H -#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/BinaryFormat/DXContainer.h" -#include "llvm/MC/StringTableBuilder.h" #include "llvm/TargetParser/Triple.h" #include @@ -47,9 +45,8 @@ struct PSVSignatureElement { // modifiable format, and can be used to serialize the data back into valid PSV // RuntimeInfo. struct PSVRuntimeInfo { - PSVRuntimeInfo() : DXConStrTabBuilder(StringTableBuilder::DXContainer) {} bool IsFinalized = false; - dxbc::PSV::v3::RuntimeInfo BaseData; + dxbc::PSV::v2::RuntimeInfo BaseData; SmallVector Resources; SmallVector InputElements; SmallVector OutputElements; @@ -67,7 +64,6 @@ struct PSVRuntimeInfo { std::array, 4> InputOutputMap; SmallVector InputPatchMap; SmallVector PatchOutputMap; - llvm::StringRef EntryName; // Serialize PSVInfo into the provided raw_ostream. The version field // specifies the data version to encode, the default value specifies encoding @@ -75,12 +71,19 @@ struct PSVRuntimeInfo { void write(raw_ostream &OS, uint32_t Version = std::numeric_limits::max()) const; - void finalize(Triple::EnvironmentType Stage); - -private: - SmallVector IndexBuffer; - SmallVector SignatureElements; - StringTableBuilder DXConStrTabBuilder; + void finalize(Triple::EnvironmentType Stage) { + IsFinalized = true; + BaseData.SigInputElements = static_cast(InputElements.size()); + BaseData.SigOutputElements = static_cast(OutputElements.size()); + BaseData.SigPatchOrPrimElements = + static_cast(PatchOrPrimElements.size()); + if (!sys::IsBigEndianHost) + return; + BaseData.swapBytes(); + BaseData.swapBytes(Stage); + for (auto &Res : Resources) + Res.swapBytes(); + } }; class Signature { diff --git a/llvm/include/llvm/MC/StringTableBuilder.h b/llvm/include/llvm/MC/StringTableBuilder.h index a738683548cf..4ee421e22c17 100644 --- a/llvm/include/llvm/MC/StringTableBuilder.h +++ b/llvm/include/llvm/MC/StringTableBuilder.h @@ -74,8 +74,12 @@ public: /// Check if a string is contained in the string table. Since this class /// doesn't store the string values, this function can be used to check if /// storage needs to be done prior to adding the string. - bool contains(StringRef S) const { return contains(CachedHashStringRef(S)); } - bool contains(CachedHashStringRef S) const { return StringIndexMap.count(S); } + bool contains(StringRef S) const { + return contains(CachedHashStringRef(S)); + } + bool contains(CachedHashStringRef S) const { + return StringIndexMap.count(S); + } size_t getSize() const { return Size; } void clear(); diff --git a/llvm/include/llvm/Object/DXContainer.h b/llvm/include/llvm/Object/DXContainer.h index 19c83ba6c6e8..b6e3d321da24 100644 --- a/llvm/include/llvm/Object/DXContainer.h +++ b/llvm/include/llvm/Object/DXContainer.h @@ -125,8 +125,7 @@ class PSVRuntimeInfo { uint32_t Size; using InfoStruct = std::variant; + dxbc::PSV::v1::RuntimeInfo, dxbc::PSV::v2::RuntimeInfo>; InfoStruct BasicInfo; ResourceArray Resources; StringRef StringTable; @@ -152,11 +151,9 @@ public: ResourceArray getResources() const { return Resources; } uint32_t getVersion() const { - return Size >= sizeof(dxbc::PSV::v3::RuntimeInfo) - ? 3 - : (Size >= sizeof(dxbc::PSV::v2::RuntimeInfo) ? 2 - : (Size >= sizeof(dxbc::PSV::v1::RuntimeInfo)) ? 1 - : 0); + return Size >= sizeof(dxbc::PSV::v2::RuntimeInfo) + ? 2 + : (Size >= sizeof(dxbc::PSV::v1::RuntimeInfo) ? 1 : 0); } uint32_t getResourceStride() const { return Resources.Stride; } @@ -164,11 +161,6 @@ public: const InfoStruct &getInfo() const { return BasicInfo; } template const T *getInfoAs() const { - if (const auto *P = std::get_if(&BasicInfo)) - return static_cast(P); - if (std::is_same::value) - return nullptr; - if (const auto *P = std::get_if(&BasicInfo)) return static_cast(P); if (std::is_same::value) diff --git a/llvm/include/llvm/ObjectYAML/DXContainerYAML.h b/llvm/include/llvm/ObjectYAML/DXContainerYAML.h index 9c4d9e19f11b..f7f8d5e6bf47 100644 --- a/llvm/include/llvm/ObjectYAML/DXContainerYAML.h +++ b/llvm/include/llvm/ObjectYAML/DXContainerYAML.h @@ -107,7 +107,7 @@ struct PSVInfo { // the format. uint32_t Version; - dxbc::PSV::v3::RuntimeInfo Info; + dxbc::PSV::v2::RuntimeInfo Info; uint32_t ResourceStride; SmallVector Resources; SmallVector SigInputElements; @@ -121,15 +121,12 @@ struct PSVInfo { MaskVector InputPatchMap; MaskVector PatchOutputMap; - StringRef EntryName; - void mapInfoForVersion(yaml::IO &IO); PSVInfo(); PSVInfo(const dxbc::PSV::v0::RuntimeInfo *P, uint16_t Stage); PSVInfo(const dxbc::PSV::v1::RuntimeInfo *P); PSVInfo(const dxbc::PSV::v2::RuntimeInfo *P); - PSVInfo(const dxbc::PSV::v3::RuntimeInfo *P, StringRef StringTable); }; struct SignatureParameter { diff --git a/llvm/lib/MC/DXContainerPSVInfo.cpp b/llvm/lib/MC/DXContainerPSVInfo.cpp index aeff69380139..48182fcd31df 100644 --- a/llvm/lib/MC/DXContainerPSVInfo.cpp +++ b/llvm/lib/MC/DXContainerPSVInfo.cpp @@ -81,18 +81,13 @@ void PSVRuntimeInfo::write(raw_ostream &OS, uint32_t Version) const { BindingSize = sizeof(dxbc::PSV::v0::ResourceBindInfo); break; case 2: - InfoSize = sizeof(dxbc::PSV::v2::RuntimeInfo); - BindingSize = sizeof(dxbc::PSV::v2::ResourceBindInfo); - break; - case 3: default: - InfoSize = sizeof(dxbc::PSV::v3::RuntimeInfo); + InfoSize = sizeof(dxbc::PSV::v2::RuntimeInfo); BindingSize = sizeof(dxbc::PSV::v2::ResourceBindInfo); } - // Write the size of the info. - support::endian::write(OS, InfoSize, llvm::endianness::little); + support::endian::write(OS, InfoSize, llvm::endianness::little); // Write the info itself. OS.write(reinterpret_cast(&BaseData), InfoSize); @@ -109,12 +104,32 @@ void PSVRuntimeInfo::write(raw_ostream &OS, uint32_t Version) const { if (Version == 0) return; - support::endian::write(OS, - static_cast(DXConStrTabBuilder.getSize()), + StringTableBuilder StrTabBuilder((StringTableBuilder::DXContainer)); + SmallVector IndexBuffer; + SmallVector SignatureElements; + SmallVector SemanticNames; + + ProcessElementList(StrTabBuilder, IndexBuffer, SignatureElements, + SemanticNames, InputElements); + ProcessElementList(StrTabBuilder, IndexBuffer, SignatureElements, + SemanticNames, OutputElements); + ProcessElementList(StrTabBuilder, IndexBuffer, SignatureElements, + SemanticNames, PatchOrPrimElements); + + StrTabBuilder.finalize(); + for (auto ElAndName : zip(SignatureElements, SemanticNames)) { + v0::SignatureElement &El = std::get<0>(ElAndName); + StringRef Name = std::get<1>(ElAndName); + El.NameOffset = static_cast(StrTabBuilder.getOffset(Name)); + if (sys::IsBigEndianHost) + El.swapBytes(); + } + + support::endian::write(OS, static_cast(StrTabBuilder.getSize()), llvm::endianness::little); // Write the string table. - DXConStrTabBuilder.write(OS); + StrTabBuilder.write(OS); // Write the index table size, then table. support::endian::write(OS, static_cast(IndexBuffer.size()), @@ -147,46 +162,6 @@ void PSVRuntimeInfo::write(raw_ostream &OS, uint32_t Version) const { llvm::endianness::little); } -void PSVRuntimeInfo::finalize(Triple::EnvironmentType Stage) { - IsFinalized = true; - BaseData.SigInputElements = static_cast(InputElements.size()); - BaseData.SigOutputElements = static_cast(OutputElements.size()); - BaseData.SigPatchOrPrimElements = - static_cast(PatchOrPrimElements.size()); - - SmallVector SemanticNames; - - // Build a string table and set associated offsets to be written when - // write() is called - ProcessElementList(DXConStrTabBuilder, IndexBuffer, SignatureElements, - SemanticNames, InputElements); - ProcessElementList(DXConStrTabBuilder, IndexBuffer, SignatureElements, - SemanticNames, OutputElements); - ProcessElementList(DXConStrTabBuilder, IndexBuffer, SignatureElements, - SemanticNames, PatchOrPrimElements); - - DXConStrTabBuilder.add(EntryName); - - DXConStrTabBuilder.finalize(); - for (auto ElAndName : zip(SignatureElements, SemanticNames)) { - llvm::dxbc::PSV::v0::SignatureElement &El = std::get<0>(ElAndName); - StringRef Name = std::get<1>(ElAndName); - El.NameOffset = static_cast(DXConStrTabBuilder.getOffset(Name)); - if (sys::IsBigEndianHost) - El.swapBytes(); - } - - BaseData.EntryNameOffset = - static_cast(DXConStrTabBuilder.getOffset(EntryName)); - - if (!sys::IsBigEndianHost) - return; - BaseData.swapBytes(); - BaseData.swapBytes(Stage); - for (auto &Res : Resources) - Res.swapBytes(); -} - void Signature::write(raw_ostream &OS) { SmallVector SigParams; SigParams.reserve(Params.size()); diff --git a/llvm/lib/Object/DXContainer.cpp b/llvm/lib/Object/DXContainer.cpp index 3b1a6203a1f8..935749afe338 100644 --- a/llvm/lib/Object/DXContainer.cpp +++ b/llvm/lib/Object/DXContainer.cpp @@ -247,14 +247,7 @@ Error DirectX::PSVRuntimeInfo::parse(uint16_t ShaderKind) { const uint32_t PSVVersion = getVersion(); // Detect the PSVVersion by looking at the size field. - if (PSVVersion == 3) { - v3::RuntimeInfo Info; - if (Error Err = readStruct(PSVInfoData, Current, Info)) - return Err; - if (sys::IsBigEndianHost) - Info.swapBytes(ShaderStage); - BasicInfo = Info; - } else if (PSVVersion == 2) { + if (PSVVersion == 2) { v2::RuntimeInfo Info; if (Error Err = readStruct(PSVInfoData, Current, Info)) return Err; @@ -432,8 +425,6 @@ Error DirectX::PSVRuntimeInfo::parse(uint16_t ShaderKind) { } uint8_t DirectX::PSVRuntimeInfo::getSigInputCount() const { - if (const auto *P = std::get_if(&BasicInfo)) - return P->SigInputElements; if (const auto *P = std::get_if(&BasicInfo)) return P->SigInputElements; if (const auto *P = std::get_if(&BasicInfo)) @@ -442,8 +433,6 @@ uint8_t DirectX::PSVRuntimeInfo::getSigInputCount() const { } uint8_t DirectX::PSVRuntimeInfo::getSigOutputCount() const { - if (const auto *P = std::get_if(&BasicInfo)) - return P->SigOutputElements; if (const auto *P = std::get_if(&BasicInfo)) return P->SigOutputElements; if (const auto *P = std::get_if(&BasicInfo)) @@ -452,8 +441,6 @@ uint8_t DirectX::PSVRuntimeInfo::getSigOutputCount() const { } uint8_t DirectX::PSVRuntimeInfo::getSigPatchOrPrimCount() const { - if (const auto *P = std::get_if(&BasicInfo)) - return P->SigPatchOrPrimElements; if (const auto *P = std::get_if(&BasicInfo)) return P->SigPatchOrPrimElements; if (const auto *P = std::get_if(&BasicInfo)) diff --git a/llvm/lib/ObjectYAML/DXContainerEmitter.cpp b/llvm/lib/ObjectYAML/DXContainerEmitter.cpp index f3a518df3175..09a5e41c7123 100644 --- a/llvm/lib/ObjectYAML/DXContainerEmitter.cpp +++ b/llvm/lib/ObjectYAML/DXContainerEmitter.cpp @@ -198,9 +198,8 @@ void DXContainerWriter::writeParts(raw_ostream &OS) { if (!P.Info.has_value()) continue; mcdxbc::PSVRuntimeInfo PSV; - memcpy(&PSV.BaseData, &P.Info->Info, sizeof(dxbc::PSV::v3::RuntimeInfo)); + memcpy(&PSV.BaseData, &P.Info->Info, sizeof(dxbc::PSV::v2::RuntimeInfo)); PSV.Resources = P.Info->Resources; - PSV.EntryName = P.Info->EntryName; for (auto El : P.Info->SigInputElements) PSV.InputElements.push_back(mcdxbc::PSVSignatureElement{ diff --git a/llvm/lib/ObjectYAML/DXContainerYAML.cpp b/llvm/lib/ObjectYAML/DXContainerYAML.cpp index 38063670aee6..a6871e7855e4 100644 --- a/llvm/lib/ObjectYAML/DXContainerYAML.cpp +++ b/llvm/lib/ObjectYAML/DXContainerYAML.cpp @@ -74,16 +74,6 @@ DXContainerYAML::PSVInfo::PSVInfo(const dxbc::PSV::v2::RuntimeInfo *P) memcpy(&Info, P, sizeof(dxbc::PSV::v2::RuntimeInfo)); } -DXContainerYAML::PSVInfo::PSVInfo(const dxbc::PSV::v3::RuntimeInfo *P, - StringRef StringTable) - : Version(3), - EntryName(StringTable.substr(P->EntryNameOffset, - StringTable.find('\0', P->EntryNameOffset) - - P->EntryNameOffset)) { - memset(&Info, 0, sizeof(Info)); - memcpy(&Info, P, sizeof(dxbc::PSV::v3::RuntimeInfo)); -} - namespace yaml { void MappingTraits::mapping( @@ -358,11 +348,6 @@ void DXContainerYAML::PSVInfo::mapInfoForVersion(yaml::IO &IO) { IO.mapRequired("NumThreadsX", Info.NumThreadsX); IO.mapRequired("NumThreadsY", Info.NumThreadsY); IO.mapRequired("NumThreadsZ", Info.NumThreadsZ); - - if (Version == 2) - return; - - IO.mapRequired("EntryName", EntryName); } } // namespace llvm diff --git a/llvm/test/ObjectYAML/DXContainer/PSVv3-amplification.yaml b/llvm/test/ObjectYAML/DXContainer/PSVv3-amplification.yaml deleted file mode 100644 index 09885bd529f0..000000000000 --- a/llvm/test/ObjectYAML/DXContainer/PSVv3-amplification.yaml +++ /dev/null @@ -1,97 +0,0 @@ -# RUN: yaml2obj %s | obj2yaml | FileCheck %s - ---- !dxcontainer -Header: - Hash: [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ] - Version: - Major: 1 - Minor: 0 - PartCount: 2 -Parts: - - Name: PSV0 - Size: 144 - PSVInfo: - Version: 3 - ShaderStage: 14 - PayloadSizeInBytes: 4092 - MinimumWaveLaneCount: 0 - MaximumWaveLaneCount: 4294967295 - UsesViewID: 0 - SigInputVectors: 0 - SigOutputVectors: [ 8, 16, 32, 64 ] - NumThreadsX: 512 - NumThreadsY: 1024 - NumThreadsZ: 2048 - EntryName: ASEntry - ResourceStride: 24 - Resources: - - Type: 1 - Space: 2 - LowerBound: 3 - UpperBound: 4 - Kind: 5 - Flags: 6 - - Type: 128 - Space: 32768 - LowerBound: 8388608 - UpperBound: 2147483648 - Kind: 65535 - Flags: 16776960 - SigInputElements: [] - SigOutputElements: [] - SigPatchOrPrimElements: [] - InputOutputMap: - - [ ] - - [ ] - - [ ] - - [ ] - - Name: DXIL - Size: 24 - Program: - MajorVersion: 6 - MinorVersion: 0 - ShaderKind: 14 - Size: 6 - DXILMajorVersion: 0 - DXILMinorVersion: 1 - DXILSize: 0 -... - -# CHECK: Name: PSV0 -# CHECK: PSVInfo: -# CHECK-NEXT: Version: 3 -# CHECK-NEXT: ShaderStage: 14 -# CHECK-NEXT: PayloadSizeInBytes: 4092 -# CHECK-NEXT: MinimumWaveLaneCount: 0 -# CHECK-NEXT: MaximumWaveLaneCount: 4294967295 -# CHECK-NEXT: UsesViewID: 0 -# CHECK-NEXT: SigInputVectors: 0 -# CHECK-NEXT: SigOutputVectors: [ 8, 16, 32, 64 ] -# CHECK-NEXT: NumThreadsX: 512 -# CHECK-NEXT: NumThreadsY: 1024 -# CHECK-NEXT: NumThreadsZ: 2048 -# CHECK-NEXT: EntryName: ASEntry -# CHECK-NEXT: ResourceStride: 24 -# CHECK-NEXT: Resources: -# CHECK-NEXT: - Type: 1 -# CHECK-NEXT: Space: 2 -# CHECK-NEXT: LowerBound: 3 -# CHECK-NEXT: UpperBound: 4 -# CHECK-NEXT: Kind: 5 -# CHECK-NEXT: Flags: 6 -# CHECK-NEXT: - Type: 128 -# CHECK-NEXT: Space: 32768 -# CHECK-NEXT: LowerBound: 8388608 -# CHECK-NEXT: UpperBound: 2147483648 -# CHECK-NEXT: Kind: 65535 -# CHECK-NEXT: Flags: 16776960 -# CHECK-NEXT: SigInputElements: [] -# CHECK-NEXT: SigOutputElements: [] -# CHECK-NEXT: SigPatchOrPrimElements: [] -# CHECK-NEXT: InputOutputMap: -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: Name diff --git a/llvm/test/ObjectYAML/DXContainer/PSVv3-compute.yaml b/llvm/test/ObjectYAML/DXContainer/PSVv3-compute.yaml deleted file mode 100644 index ee6fb112c772..000000000000 --- a/llvm/test/ObjectYAML/DXContainer/PSVv3-compute.yaml +++ /dev/null @@ -1,95 +0,0 @@ -# RUN: yaml2obj %s | obj2yaml | FileCheck %s - ---- !dxcontainer -Header: - Hash: [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ] - Version: - Major: 1 - Minor: 0 - PartCount: 2 -Parts: - - Name: PSV0 - Size: 144 - PSVInfo: - Version: 3 - ShaderStage: 5 - MinimumWaveLaneCount: 0 - MaximumWaveLaneCount: 4294967295 - UsesViewID: 0 - SigInputVectors: 0 - SigOutputVectors: [ 8, 16, 32, 64 ] - NumThreadsX: 512 - NumThreadsY: 1024 - NumThreadsZ: 2048 - EntryName: CSEntry - ResourceStride: 24 - Resources: - - Type: 1 - Space: 2 - LowerBound: 3 - UpperBound: 4 - Kind: 5 - Flags: 6 - - Type: 128 - Space: 32768 - LowerBound: 8388608 - UpperBound: 2147483648 - Kind: 65535 - Flags: 16776960 - SigInputElements: [] - SigOutputElements: [] - SigPatchOrPrimElements: [] - InputOutputMap: - - [ ] - - [ ] - - [ ] - - [ ] - - Name: DXIL - Size: 24 - Program: - MajorVersion: 6 - MinorVersion: 0 - ShaderKind: 5 - Size: 6 - DXILMajorVersion: 0 - DXILMinorVersion: 1 - DXILSize: 0 -... - -# CHECK: Name: PSV0 -# CHECK: PSVInfo: -# CHECK-NEXT: Version: 3 -# CHECK-NEXT: ShaderStage: 5 -# CHECK-NEXT: MinimumWaveLaneCount: 0 -# CHECK-NEXT: MaximumWaveLaneCount: 4294967295 -# CHECK-NEXT: UsesViewID: 0 -# CHECK-NEXT: SigInputVectors: 0 -# CHECK-NEXT: SigOutputVectors: [ 8, 16, 32, 64 ] -# CHECK-NEXT: NumThreadsX: 512 -# CHECK-NEXT: NumThreadsY: 1024 -# CHECK-NEXT: NumThreadsZ: 2048 -# CHECK-NEXT: EntryName: CSEntry -# CHECK-NEXT: ResourceStride: 24 -# CHECK-NEXT: Resources: -# CHECK-NEXT: - Type: 1 -# CHECK-NEXT: Space: 2 -# CHECK-NEXT: LowerBound: 3 -# CHECK-NEXT: UpperBound: 4 -# CHECK-NEXT: Kind: 5 -# CHECK-NEXT: Flags: 6 -# CHECK-NEXT: - Type: 128 -# CHECK-NEXT: Space: 32768 -# CHECK-NEXT: LowerBound: 8388608 -# CHECK-NEXT: UpperBound: 2147483648 -# CHECK-NEXT: Kind: 65535 -# CHECK-NEXT: Flags: 16776960 -# CHECK-NEXT: SigInputElements: [] -# CHECK-NEXT: SigOutputElements: [] -# CHECK-NEXT: SigPatchOrPrimElements: [] -# CHECK-NEXT: InputOutputMap: -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: Name diff --git a/llvm/test/ObjectYAML/DXContainer/PSVv3-domain.yaml b/llvm/test/ObjectYAML/DXContainer/PSVv3-domain.yaml deleted file mode 100644 index dd367deae88e..000000000000 --- a/llvm/test/ObjectYAML/DXContainer/PSVv3-domain.yaml +++ /dev/null @@ -1,105 +0,0 @@ -# RUN: yaml2obj %s | obj2yaml | FileCheck %s - ---- !dxcontainer -Header: - Hash: [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ] - Version: - Major: 1 - Minor: 0 - PartCount: 2 -Parts: - - Name: PSV0 - Size: 144 - PSVInfo: - Version: 3 - ShaderStage: 4 - InputControlPointCount: 1024 - OutputPositionPresent: 1 - TessellatorDomain: 2056 - MinimumWaveLaneCount: 0 - MaximumWaveLaneCount: 4294967295 - UsesViewID: 0 - SigPatchConstOrPrimVectors: 0 - SigInputVectors: 0 - SigOutputVectors: [ 0, 16, 32, 64 ] - NumThreadsX: 512 - NumThreadsY: 1024 - NumThreadsZ: 2048 - EntryName: DSEntry - ResourceStride: 24 - Resources: - - Type: 1 - Space: 2 - LowerBound: 3 - UpperBound: 4 - Kind: 5 - Flags: 6 - - Type: 128 - Space: 32768 - LowerBound: 8388608 - UpperBound: 2147483648 - Kind: 65535 - Flags: 16776960 - SigInputElements: [] - SigOutputElements: [] - SigPatchOrPrimElements: [] - InputOutputMap: - - [ ] - - [ ] - - [ ] - - [ ] - PatchOutputMap: [] - - Name: DXIL - Size: 24 - Program: - MajorVersion: 6 - MinorVersion: 0 - ShaderKind: 4 - Size: 6 - DXILMajorVersion: 0 - DXILMinorVersion: 1 - DXILSize: 0 -... - -# CHECK: Name: PSV0 -# CHECK: PSVInfo: -# CHECK-NEXT: Version: 3 -# CHECK-NEXT: ShaderStage: 4 -# CHECK-NEXT: InputControlPointCount: 1024 -# CHECK-NEXT: OutputPositionPresent: 1 -# CHECK-NEXT: TessellatorDomain: 2056 -# CHECK-NEXT: MinimumWaveLaneCount: 0 -# CHECK-NEXT: MaximumWaveLaneCount: 4294967295 -# CHECK-NEXT: UsesViewID: 0 -# CHECK-NEXT: SigPatchConstOrPrimVectors: 0 -# CHECK-NEXT: SigInputVectors: 0 -# CHECK-NEXT: SigOutputVectors: [ 0, 16, 32, 64 ] -# CHECK-NEXT: NumThreadsX: 512 -# CHECK-NEXT: NumThreadsY: 1024 -# CHECK-NEXT: NumThreadsZ: 2048 -# CHECK-NEXT: EntryName: DSEntry -# CHECK-NEXT: ResourceStride: 24 -# CHECK-NEXT: Resources: -# CHECK-NEXT: - Type: 1 -# CHECK-NEXT: Space: 2 -# CHECK-NEXT: LowerBound: 3 -# CHECK-NEXT: UpperBound: 4 -# CHECK-NEXT: Kind: 5 -# CHECK-NEXT: Flags: 6 -# CHECK-NEXT: - Type: 128 -# CHECK-NEXT: Space: 32768 -# CHECK-NEXT: LowerBound: 8388608 -# CHECK-NEXT: UpperBound: 2147483648 -# CHECK-NEXT: Kind: 65535 -# CHECK-NEXT: Flags: 16776960 -# CHECK-NEXT: SigInputElements: [] -# CHECK-NEXT: SigOutputElements: [] -# CHECK-NEXT: SigPatchOrPrimElements: [] -# CHECK-NEXT: InputOutputMap: -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: PatchOutputMap: [ ] -# CHECK-NEXT: Name diff --git a/llvm/test/ObjectYAML/DXContainer/PSVv3-geometry.yaml b/llvm/test/ObjectYAML/DXContainer/PSVv3-geometry.yaml deleted file mode 100644 index 4c7680b63b02..000000000000 --- a/llvm/test/ObjectYAML/DXContainer/PSVv3-geometry.yaml +++ /dev/null @@ -1,105 +0,0 @@ -# RUN: yaml2obj %s | obj2yaml | FileCheck %s - ---- !dxcontainer -Header: - Hash: [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ] - Version: - Major: 1 - Minor: 0 - PartCount: 2 -Parts: - - Name: PSV0 - Size: 144 - PSVInfo: - Version: 3 - ShaderStage: 2 - InputPrimitive: 1024 - OutputTopology: 4096 - OutputStreamMask: 2056 - OutputPositionPresent: 1 - MinimumWaveLaneCount: 0 - MaximumWaveLaneCount: 4294967295 - UsesViewID: 0 - MaxVertexCount: 4096 - SigInputVectors: 0 - SigOutputVectors: [ 8, 16, 32, 64 ] - NumThreadsX: 512 - NumThreadsY: 1024 - NumThreadsZ: 2048 - EntryName: GSEntry - ResourceStride: 24 - Resources: - - Type: 1 - Space: 2 - LowerBound: 3 - UpperBound: 4 - Kind: 5 - Flags: 6 - - Type: 128 - Space: 32768 - LowerBound: 8388608 - UpperBound: 2147483648 - Kind: 65535 - Flags: 16776960 - SigInputElements: [] - SigOutputElements: [] - SigPatchOrPrimElements: [] - InputOutputMap: - - [ ] - - [ ] - - [ ] - - [ ] - - Name: DXIL - Size: 24 - Program: - MajorVersion: 6 - MinorVersion: 0 - ShaderKind: 2 - Size: 6 - DXILMajorVersion: 0 - DXILMinorVersion: 1 - DXILSize: 0 -... - -# CHECK: Name: PSV0 -# CHECK: PSVInfo: -# CHECK-NEXT: Version: 3 -# CHECK-NEXT: ShaderStage: 2 -# CHECK-NEXT: InputPrimitive: 1024 -# CHECK-NEXT: OutputTopology: 4096 -# CHECK-NEXT: OutputStreamMask: 2056 -# CHECK-NEXT: OutputPositionPresent: 1 -# CHECK-NEXT: MinimumWaveLaneCount: 0 -# CHECK-NEXT: MaximumWaveLaneCount: 4294967295 -# CHECK-NEXT: UsesViewID: 0 -# CHECK-NEXT: MaxVertexCount: 4096 -# CHECK-NEXT: SigInputVectors: 0 -# CHECK-NEXT: SigOutputVectors: [ 8, 16, 32, 64 ] -# CHECK-NEXT: NumThreadsX: 512 -# CHECK-NEXT: NumThreadsY: 1024 -# CHECK-NEXT: NumThreadsZ: 2048 -# CHECK-NEXT: EntryName: GSEntry -# CHECK-NEXT: ResourceStride: 24 -# CHECK-NEXT: Resources: -# CHECK-NEXT: - Type: 1 -# CHECK-NEXT: Space: 2 -# CHECK-NEXT: LowerBound: 3 -# CHECK-NEXT: UpperBound: 4 -# CHECK-NEXT: Kind: 5 -# CHECK-NEXT: Flags: 6 -# CHECK-NEXT: - Type: 128 -# CHECK-NEXT: Space: 32768 -# CHECK-NEXT: LowerBound: 8388608 -# CHECK-NEXT: UpperBound: 2147483648 -# CHECK-NEXT: Kind: 65535 -# CHECK-NEXT: Flags: 16776960 -# CHECK-NEXT: SigInputElements: [] -# CHECK-NEXT: SigOutputElements: [] -# CHECK-NEXT: SigPatchOrPrimElements: [] -# CHECK-NEXT: InputOutputMap: -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: Name diff --git a/llvm/test/ObjectYAML/DXContainer/PSVv3-hull.yaml b/llvm/test/ObjectYAML/DXContainer/PSVv3-hull.yaml deleted file mode 100644 index 3bbad8a9b0ee..000000000000 --- a/llvm/test/ObjectYAML/DXContainer/PSVv3-hull.yaml +++ /dev/null @@ -1,107 +0,0 @@ -# RUN: yaml2obj %s | obj2yaml | FileCheck %s - ---- !dxcontainer -Header: - Hash: [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ] - Version: - Major: 1 - Minor: 0 - PartCount: 2 -Parts: - - Name: PSV0 - Size: 144 - PSVInfo: - Version: 3 - ShaderStage: 3 - InputControlPointCount: 1024 - OutputControlPointCount: 4096 - TessellatorDomain: 2056 - TessellatorOutputPrimitive: 8192 - MinimumWaveLaneCount: 0 - MaximumWaveLaneCount: 4294967295 - UsesViewID: 0 - SigPatchConstOrPrimVectors: 0 - SigInputVectors: 0 - SigOutputVectors: [ 0, 16, 32, 64 ] - NumThreadsX: 512 - NumThreadsY: 1024 - NumThreadsZ: 2048 - EntryName: HSEntry - ResourceStride: 24 - Resources: - - Type: 1 - Space: 2 - LowerBound: 3 - UpperBound: 4 - Kind: 5 - Flags: 6 - - Type: 128 - Space: 32768 - LowerBound: 8388608 - UpperBound: 2147483648 - Kind: 65535 - Flags: 16776960 - SigInputElements: [] - SigOutputElements: [] - SigPatchOrPrimElements: [] - InputOutputMap: - - [ ] - - [ ] - - [ ] - - [ ] - InputPatchMap: [] - - Name: DXIL - Size: 24 - Program: - MajorVersion: 6 - MinorVersion: 0 - ShaderKind: 3 - Size: 6 - DXILMajorVersion: 0 - DXILMinorVersion: 1 - DXILSize: 0 -... - -# CHECK: Name: PSV0 -# CHECK: PSVInfo: -# CHECK-NEXT: Version: 3 -# CHECK-NEXT: ShaderStage: 3 -# CHECK-NEXT: InputControlPointCount: 1024 -# CHECK-NEXT: OutputControlPointCount: 4096 -# CHECK-NEXT: TessellatorDomain: 2056 -# CHECK-NEXT: TessellatorOutputPrimitive: 8192 -# CHECK-NEXT: MinimumWaveLaneCount: 0 -# CHECK-NEXT: MaximumWaveLaneCount: 4294967295 -# CHECK-NEXT: UsesViewID: 0 -# CHECK-NEXT: SigPatchConstOrPrimVectors: 0 -# CHECK-NEXT: SigInputVectors: 0 -# CHECK-NEXT: SigOutputVectors: [ 0, 16, 32, 64 ] -# CHECK-NEXT: NumThreadsX: 512 -# CHECK-NEXT: NumThreadsY: 1024 -# CHECK-NEXT: NumThreadsZ: 2048 -# CHECK-NEXT: EntryName: HSEntry -# CHECK-NEXT: ResourceStride: 24 -# CHECK-NEXT: Resources: -# CHECK-NEXT: - Type: 1 -# CHECK-NEXT: Space: 2 -# CHECK-NEXT: LowerBound: 3 -# CHECK-NEXT: UpperBound: 4 -# CHECK-NEXT: Kind: 5 -# CHECK-NEXT: Flags: 6 -# CHECK-NEXT: - Type: 128 -# CHECK-NEXT: Space: 32768 -# CHECK-NEXT: LowerBound: 8388608 -# CHECK-NEXT: UpperBound: 2147483648 -# CHECK-NEXT: Kind: 65535 -# CHECK-NEXT: Flags: 16776960 -# CHECK-NEXT: SigInputElements: [] -# CHECK-NEXT: SigOutputElements: [] -# CHECK-NEXT: SigPatchOrPrimElements: [] -# CHECK-NEXT: InputOutputMap: -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: InputPatchMap: [ ] -# CHECK-NEXT: Name diff --git a/llvm/test/ObjectYAML/DXContainer/PSVv3-mesh.yaml b/llvm/test/ObjectYAML/DXContainer/PSVv3-mesh.yaml deleted file mode 100644 index c5ea1fcf0780..000000000000 --- a/llvm/test/ObjectYAML/DXContainer/PSVv3-mesh.yaml +++ /dev/null @@ -1,109 +0,0 @@ -# RUN: yaml2obj %s | obj2yaml | FileCheck %s - ---- !dxcontainer -Header: - Hash: [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ] - Version: - Major: 1 - Minor: 0 - PartCount: 2 -Parts: - - Name: PSV0 - Size: 144 - PSVInfo: - Version: 3 - ShaderStage: 13 - GroupSharedBytesUsed: 1024 - GroupSharedBytesDependentOnViewID: 2056 - PayloadSizeInBytes: 4092 - MaxOutputVertices: 8196 - MaxOutputPrimitives: 4092 - MinimumWaveLaneCount: 0 - MaximumWaveLaneCount: 4294967295 - UsesViewID: 0 - SigPrimVectors: 128 - MeshOutputTopology: 16 - SigInputVectors: 0 - SigOutputVectors: [ 8, 16, 32, 64 ] - NumThreadsX: 512 - NumThreadsY: 1024 - NumThreadsZ: 2048 - EntryName: MSEntry - ResourceStride: 24 - Resources: - - Type: 1 - Space: 2 - LowerBound: 3 - UpperBound: 4 - Kind: 5 - Flags: 6 - - Type: 128 - Space: 32768 - LowerBound: 8388608 - UpperBound: 2147483648 - Kind: 65535 - Flags: 16776960 - SigInputElements: [] - SigOutputElements: [] - SigPatchOrPrimElements: [] - InputOutputMap: - - [ ] - - [ ] - - [ ] - - [ ] - - Name: DXIL - Size: 24 - Program: - MajorVersion: 6 - MinorVersion: 0 - ShaderKind: 13 - Size: 6 - DXILMajorVersion: 0 - DXILMinorVersion: 1 - DXILSize: 0 -... - -# CHECK: Name: PSV0 -# CHECK: PSVInfo: -# CHECK-NEXT: Version: 3 -# CHECK-NEXT: ShaderStage: 13 -# CHECK-NEXT: GroupSharedBytesUsed: 1024 -# CHECK-NEXT: GroupSharedBytesDependentOnViewID: 2056 -# CHECK-NEXT: PayloadSizeInBytes: 4092 -# CHECK-NEXT: MaxOutputVertices: 8196 -# CHECK-NEXT: MaxOutputPrimitives: 4092 -# CHECK-NEXT: MinimumWaveLaneCount: 0 -# CHECK-NEXT: MaximumWaveLaneCount: 4294967295 -# CHECK-NEXT: UsesViewID: 0 -# CHECK-NEXT: SigPrimVectors: 128 -# CHECK-NEXT: MeshOutputTopology: 16 -# CHECK-NEXT: SigInputVectors: 0 -# CHECK-NEXT: SigOutputVectors: [ 8, 16, 32, 64 ] -# CHECK-NEXT: NumThreadsX: 512 -# CHECK-NEXT: NumThreadsY: 1024 -# CHECK-NEXT: NumThreadsZ: 2048 -# CHECK-NEXT: EntryName: MSEntry -# CHECK-NEXT: ResourceStride: 24 -# CHECK-NEXT: Resources: -# CHECK-NEXT: - Type: 1 -# CHECK-NEXT: Space: 2 -# CHECK-NEXT: LowerBound: 3 -# CHECK-NEXT: UpperBound: 4 -# CHECK-NEXT: Kind: 5 -# CHECK-NEXT: Flags: 6 -# CHECK-NEXT: - Type: 128 -# CHECK-NEXT: Space: 32768 -# CHECK-NEXT: LowerBound: 8388608 -# CHECK-NEXT: UpperBound: 2147483648 -# CHECK-NEXT: Kind: 65535 -# CHECK-NEXT: Flags: 16776960 -# CHECK-NEXT: SigInputElements: [] -# CHECK-NEXT: SigOutputElements: [] -# CHECK-NEXT: SigPatchOrPrimElements: [] -# CHECK-NEXT: InputOutputMap: -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: Name diff --git a/llvm/test/ObjectYAML/DXContainer/PSVv3-pixel.yaml b/llvm/test/ObjectYAML/DXContainer/PSVv3-pixel.yaml deleted file mode 100644 index b28d5ec8074d..000000000000 --- a/llvm/test/ObjectYAML/DXContainer/PSVv3-pixel.yaml +++ /dev/null @@ -1,99 +0,0 @@ -# RUN: yaml2obj %s | obj2yaml | FileCheck %s - ---- !dxcontainer -Header: - Hash: [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ] - Version: - Major: 1 - Minor: 0 - PartCount: 2 -Parts: - - Name: PSV0 - Size: 144 - PSVInfo: - Version: 3 - ShaderStage: 0 - DepthOutput: 7 - SampleFrequency: 96 - MinimumWaveLaneCount: 0 - MaximumWaveLaneCount: 4294967295 - UsesViewID: 0 - SigInputVectors: 0 - SigOutputVectors: [ 8, 16, 32, 64 ] - NumThreadsX: 512 - NumThreadsY: 1024 - NumThreadsZ: 2048 - EntryName: PSEntry - ResourceStride: 24 - Resources: - - Type: 1 - Space: 2 - LowerBound: 3 - UpperBound: 4 - Kind: 5 - Flags: 6 - - Type: 128 - Space: 32768 - LowerBound: 8388608 - UpperBound: 2147483648 - Kind: 65535 - Flags: 16776960 - SigInputElements: [] - SigOutputElements: [] - SigPatchOrPrimElements: [] - InputOutputMap: - - [ ] - - [ ] - - [ ] - - [ ] - - Name: DXIL - Size: 24 - Program: - MajorVersion: 6 - MinorVersion: 0 - ShaderKind: 0 - Size: 6 - DXILMajorVersion: 0 - DXILMinorVersion: 1 - DXILSize: 0 -... - -# CHECK: Name: PSV0 -# CHECK: PSVInfo: -# CHECK-NEXT: Version: 3 -# CHECK-NEXT: ShaderStage: 0 -# CHECK-NEXT: DepthOutput: 7 -# CHECK-NEXT: SampleFrequency: 96 -# CHECK-NEXT: MinimumWaveLaneCount: 0 -# CHECK-NEXT: MaximumWaveLaneCount: 4294967295 -# CHECK-NEXT: UsesViewID: 0 -# CHECK-NEXT: SigInputVectors: 0 -# CHECK-NEXT: SigOutputVectors: [ 8, 16, 32, 64 ] -# CHECK-NEXT: NumThreadsX: 512 -# CHECK-NEXT: NumThreadsY: 1024 -# CHECK-NEXT: NumThreadsZ: 2048 -# CHECK-NEXT: EntryName: PSEntry -# CHECK-NEXT: ResourceStride: 24 -# CHECK-NEXT: Resources: -# CHECK-NEXT: - Type: 1 -# CHECK-NEXT: Space: 2 -# CHECK-NEXT: LowerBound: 3 -# CHECK-NEXT: UpperBound: 4 -# CHECK-NEXT: Kind: 5 -# CHECK-NEXT: Flags: 6 -# CHECK-NEXT: - Type: 128 -# CHECK-NEXT: Space: 32768 -# CHECK-NEXT: LowerBound: 8388608 -# CHECK-NEXT: UpperBound: 2147483648 -# CHECK-NEXT: Kind: 65535 -# CHECK-NEXT: Flags: 16776960 -# CHECK-NEXT: SigInputElements: [] -# CHECK-NEXT: SigOutputElements: [] -# CHECK-NEXT: SigPatchOrPrimElements: [] -# CHECK-NEXT: InputOutputMap: -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: Name diff --git a/llvm/test/ObjectYAML/DXContainer/PSVv3-vertex.yaml b/llvm/test/ObjectYAML/DXContainer/PSVv3-vertex.yaml deleted file mode 100644 index d1fb55839931..000000000000 --- a/llvm/test/ObjectYAML/DXContainer/PSVv3-vertex.yaml +++ /dev/null @@ -1,97 +0,0 @@ -# RUN: yaml2obj %s | obj2yaml | FileCheck %s - ---- !dxcontainer -Header: - Hash: [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ] - Version: - Major: 1 - Minor: 0 - PartCount: 2 -Parts: - - Name: PSV0 - Size: 144 - PSVInfo: - Version: 3 - ShaderStage: 1 - OutputPositionPresent: 1 - MinimumWaveLaneCount: 0 - MaximumWaveLaneCount: 4294967295 - UsesViewID: 0 - SigInputVectors: 0 - SigOutputVectors: [ 8, 16, 32, 64 ] - NumThreadsX: 512 - NumThreadsY: 1024 - NumThreadsZ: 2048 - EntryName: VSEntry - ResourceStride: 24 - Resources: - - Type: 1 - Space: 2 - LowerBound: 3 - UpperBound: 4 - Kind: 5 - Flags: 6 - - Type: 128 - Space: 32768 - LowerBound: 8388608 - UpperBound: 2147483648 - Kind: 65535 - Flags: 16776960 - SigInputElements: [] - SigOutputElements: [] - SigPatchOrPrimElements: [] - InputOutputMap: - - [ ] - - [ ] - - [ ] - - [ ] - - Name: DXIL - Size: 24 - Program: - MajorVersion: 6 - MinorVersion: 0 - ShaderKind: 1 - Size: 6 - DXILMajorVersion: 0 - DXILMinorVersion: 1 - DXILSize: 0 -... - -# CHECK: Name: PSV0 -# CHECK: PSVInfo: -# CHECK-NEXT: Version: 3 -# CHECK-NEXT: ShaderStage: 1 -# CHECK-NEXT: OutputPositionPresent: 1 -# CHECK-NEXT: MinimumWaveLaneCount: 0 -# CHECK-NEXT: MaximumWaveLaneCount: 4294967295 -# CHECK-NEXT: UsesViewID: 0 -# CHECK-NEXT: SigInputVectors: 0 -# CHECK-NEXT: SigOutputVectors: [ 8, 16, 32, 64 ] -# CHECK-NEXT: NumThreadsX: 512 -# CHECK-NEXT: NumThreadsY: 1024 -# CHECK-NEXT: NumThreadsZ: 2048 -# CHECK-NEXT: EntryName: VSEntry -# CHECK-NEXT: ResourceStride: 24 -# CHECK-NEXT: Resources: -# CHECK-NEXT: - Type: 1 -# CHECK-NEXT: Space: 2 -# CHECK-NEXT: LowerBound: 3 -# CHECK-NEXT: UpperBound: 4 -# CHECK-NEXT: Kind: 5 -# CHECK-NEXT: Flags: 6 -# CHECK-NEXT: - Type: 128 -# CHECK-NEXT: Space: 32768 -# CHECK-NEXT: LowerBound: 8388608 -# CHECK-NEXT: UpperBound: 2147483648 -# CHECK-NEXT: Kind: 65535 -# CHECK-NEXT: Flags: 16776960 -# CHECK-NEXT: SigInputElements: [] -# CHECK-NEXT: SigOutputElements: [] -# CHECK-NEXT: SigPatchOrPrimElements: [] -# CHECK-NEXT: InputOutputMap: -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: Name diff --git a/llvm/tools/obj2yaml/dxcontainer2yaml.cpp b/llvm/tools/obj2yaml/dxcontainer2yaml.cpp index ec4f5c74498f..69d9b9a2f784 100644 --- a/llvm/tools/obj2yaml/dxcontainer2yaml.cpp +++ b/llvm/tools/obj2yaml/dxcontainer2yaml.cpp @@ -99,9 +99,6 @@ dumpDXContainer(MemoryBufferRef Source) { else if (const auto *P = std::get_if(&PSVInfo->getInfo())) NewPart.Info = DXContainerYAML::PSVInfo(P); - else if (const auto *P = - std::get_if(&PSVInfo->getInfo())) - NewPart.Info = DXContainerYAML::PSVInfo(P, PSVInfo->getStringTable()); NewPart.Info->ResourceStride = PSVInfo->getResourceStride(); for (auto Res : PSVInfo->getResources()) NewPart.Info->Resources.push_back(Res); -- GitLab From dc74bf7a5412df82223f7062d9a6b814abbfca45 Mon Sep 17 00:00:00 2001 From: Cyndy Ishida Date: Thu, 21 Mar 2024 15:41:49 -0700 Subject: [PATCH 008/306] Add myself as codeowner for InstallAPI & TextAPI --- .github/CODEOWNERS | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 9cf6f5d79d41..fea132c8fe78 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -119,3 +119,8 @@ clang/test/AST/Interp/ @tbaederr # Bazel build system. /utils/bazel/ @rupprecht + +# InstallAPI and TextAPI +/llvm/**/TextAPI/ @cyndyishida +/clang/**/InstallAPI/ @cyndyishida +/clang/tools/clang-installapi/ @cyndyishida -- GitLab From 06d245242e3e24cd4558f545fb5ceba0582c4f03 Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Fri, 22 Mar 2024 07:08:51 +0800 Subject: [PATCH 009/306] [RISCV] Recursively split concat_vector into smaller LMULs when lowering (#85825) This is a reimplementation of the combine added in #83035 but as a lowering instead of a combine, so we don't regress the test case added in e59f120e3a14ccdc55fcb7be996efaa768daabe0 by interfering with the strided load combine Previously the combine had to concatenate the split vectors with insert_subvector instead of concat_vectors to prevent an infinite combine loop. And the reasoning behind keeping it as a combine was because if we emitted the insert_subvector during lowering then we didn't fold away inserts of undef subvectors. However it turns out we can avoid this if we just do this in lowering and select a concat_vector directly, since we get the undef folding for free with `DAG.getNode(ISD::CONCAT_VECTOR, ...)` via foldCONCAT_VECTORS. --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 24 + .../CodeGen/RISCV/rvv/active_lane_mask.ll | 91 +- .../RISCV/rvv/combine-store-extract-crash.ll | 4 +- .../CodeGen/RISCV/rvv/extract-subvector.ll | 3 +- .../RISCV/rvv/fixed-vectors-shuffle-concat.ll | 254 ++-- .../rvv/fixed-vectors-strided-load-combine.ll | 105 +- .../CodeGen/RISCV/rvv/fpclamptosat_vec.ll | 1066 ++++++++++------- llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll | 13 +- llvm/test/CodeGen/RISCV/rvv/pr63596.ll | 37 +- 9 files changed, 844 insertions(+), 753 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index a3ebfb34ad7a..71059b5bdc0f 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -6611,6 +6611,30 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op, // better than going through the stack, as the default expansion does. SDLoc DL(Op); MVT VT = Op.getSimpleValueType(); + MVT ContainerVT = VT; + if (VT.isFixedLengthVector()) + ContainerVT = ::getContainerForFixedLengthVector(DAG, VT, Subtarget); + + // Recursively split concat_vectors with more than 2 operands: + // + // concat_vector op1, op2, op3, op4 + // -> + // concat_vector (concat_vector op1, op2), (concat_vector op3, op4) + // + // This reduces the length of the chain of vslideups and allows us to + // perform the vslideups at a smaller LMUL, limited to MF2. + if (Op.getNumOperands() > 2 && + ContainerVT.bitsGE(getLMUL1VT(ContainerVT))) { + MVT HalfVT = VT.getHalfNumVectorElementsVT(); + assert(isPowerOf2_32(Op.getNumOperands())); + size_t HalfNumOps = Op.getNumOperands() / 2; + SDValue Lo = DAG.getNode(ISD::CONCAT_VECTORS, DL, HalfVT, + Op->ops().take_front(HalfNumOps)); + SDValue Hi = DAG.getNode(ISD::CONCAT_VECTORS, DL, HalfVT, + Op->ops().drop_front(HalfNumOps)); + return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi); + } + unsigned NumOpElts = Op.getOperand(0).getSimpleValueType().getVectorMinNumElements(); SDValue Vec = DAG.getUNDEF(VT); diff --git a/llvm/test/CodeGen/RISCV/rvv/active_lane_mask.ll b/llvm/test/CodeGen/RISCV/rvv/active_lane_mask.ll index 87d95d7596d4..139579b3d2a3 100644 --- a/llvm/test/CodeGen/RISCV/rvv/active_lane_mask.ll +++ b/llvm/test/CodeGen/RISCV/rvv/active_lane_mask.ll @@ -161,72 +161,71 @@ define <64 x i1> @fv64(ptr %p, i64 %index, i64 %tc) { define <128 x i1> @fv128(ptr %p, i64 %index, i64 %tc) { ; CHECK-LABEL: fv128: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetivli zero, 16, e64, m8, ta, ma ; CHECK-NEXT: lui a0, %hi(.LCPI10_0) ; CHECK-NEXT: addi a0, a0, %lo(.LCPI10_0) +; CHECK-NEXT: vsetivli zero, 16, e64, m8, ta, ma ; CHECK-NEXT: vle8.v v8, (a0) -; CHECK-NEXT: vid.v v16 -; CHECK-NEXT: vsaddu.vx v16, v16, a1 -; CHECK-NEXT: vmsltu.vx v0, v16, a2 -; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vsaddu.vx v8, v16, a1 -; CHECK-NEXT: vmsltu.vx v16, v8, a2 -; CHECK-NEXT: vsetivli zero, 4, e8, m1, tu, ma -; CHECK-NEXT: vslideup.vi v0, v16, 2 ; CHECK-NEXT: lui a0, %hi(.LCPI10_1) ; CHECK-NEXT: addi a0, a0, %lo(.LCPI10_1) -; CHECK-NEXT: vsetivli zero, 16, e64, m8, ta, ma -; CHECK-NEXT: vle8.v v8, (a0) +; CHECK-NEXT: vle8.v v9, (a0) ; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vsaddu.vx v8, v16, a1 -; CHECK-NEXT: vmsltu.vx v16, v8, a2 -; CHECK-NEXT: vsetivli zero, 6, e8, m1, tu, ma -; CHECK-NEXT: vslideup.vi v0, v16, 4 +; CHECK-NEXT: vsaddu.vx v16, v16, a1 +; CHECK-NEXT: vmsltu.vx v10, v16, a2 +; CHECK-NEXT: vsext.vf8 v16, v9 +; CHECK-NEXT: vsaddu.vx v16, v16, a1 +; CHECK-NEXT: vmsltu.vx v8, v16, a2 +; CHECK-NEXT: vsetivli zero, 4, e8, mf2, tu, ma +; CHECK-NEXT: vslideup.vi v8, v10, 2 ; CHECK-NEXT: lui a0, %hi(.LCPI10_2) ; CHECK-NEXT: addi a0, a0, %lo(.LCPI10_2) ; CHECK-NEXT: vsetivli zero, 16, e64, m8, ta, ma -; CHECK-NEXT: vle8.v v8, (a0) -; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vsaddu.vx v8, v16, a1 -; CHECK-NEXT: vmsltu.vx v16, v8, a2 -; CHECK-NEXT: vsetivli zero, 8, e8, m1, tu, ma -; CHECK-NEXT: vslideup.vi v0, v16, 6 +; CHECK-NEXT: vle8.v v9, (a0) +; CHECK-NEXT: vsext.vf8 v16, v9 +; CHECK-NEXT: vsaddu.vx v16, v16, a1 +; CHECK-NEXT: vmsltu.vx v9, v16, a2 +; CHECK-NEXT: vsetivli zero, 6, e8, mf2, tu, ma +; CHECK-NEXT: vslideup.vi v8, v9, 4 ; CHECK-NEXT: lui a0, %hi(.LCPI10_3) ; CHECK-NEXT: addi a0, a0, %lo(.LCPI10_3) ; CHECK-NEXT: vsetivli zero, 16, e64, m8, ta, ma -; CHECK-NEXT: vle8.v v8, (a0) -; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vsaddu.vx v8, v16, a1 -; CHECK-NEXT: vmsltu.vx v16, v8, a2 -; CHECK-NEXT: vsetivli zero, 10, e8, m1, tu, ma -; CHECK-NEXT: vslideup.vi v0, v16, 8 +; CHECK-NEXT: vle8.v v9, (a0) +; CHECK-NEXT: vsext.vf8 v16, v9 +; CHECK-NEXT: vsaddu.vx v16, v16, a1 +; CHECK-NEXT: vmsltu.vx v9, v16, a2 +; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma +; CHECK-NEXT: vslideup.vi v8, v9, 6 +; CHECK-NEXT: vsetivli zero, 16, e64, m8, ta, ma ; CHECK-NEXT: lui a0, %hi(.LCPI10_4) ; CHECK-NEXT: addi a0, a0, %lo(.LCPI10_4) -; CHECK-NEXT: vsetivli zero, 16, e64, m8, ta, ma -; CHECK-NEXT: vle8.v v8, (a0) -; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vsaddu.vx v8, v16, a1 -; CHECK-NEXT: vmsltu.vx v16, v8, a2 -; CHECK-NEXT: vsetivli zero, 12, e8, m1, tu, ma -; CHECK-NEXT: vslideup.vi v0, v16, 10 +; CHECK-NEXT: vle8.v v9, (a0) +; CHECK-NEXT: vid.v v16 +; CHECK-NEXT: vsaddu.vx v16, v16, a1 +; CHECK-NEXT: vmsltu.vx v0, v16, a2 +; CHECK-NEXT: vsext.vf8 v16, v9 +; CHECK-NEXT: vsaddu.vx v16, v16, a1 +; CHECK-NEXT: vmsltu.vx v9, v16, a2 +; CHECK-NEXT: vsetivli zero, 4, e8, mf2, tu, ma +; CHECK-NEXT: vslideup.vi v0, v9, 2 ; CHECK-NEXT: lui a0, %hi(.LCPI10_5) ; CHECK-NEXT: addi a0, a0, %lo(.LCPI10_5) ; CHECK-NEXT: vsetivli zero, 16, e64, m8, ta, ma -; CHECK-NEXT: vle8.v v8, (a0) -; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vsaddu.vx v8, v16, a1 -; CHECK-NEXT: vmsltu.vx v16, v8, a2 -; CHECK-NEXT: vsetivli zero, 14, e8, m1, tu, ma -; CHECK-NEXT: vslideup.vi v0, v16, 12 +; CHECK-NEXT: vle8.v v9, (a0) +; CHECK-NEXT: vsext.vf8 v16, v9 +; CHECK-NEXT: vsaddu.vx v16, v16, a1 +; CHECK-NEXT: vmsltu.vx v9, v16, a2 +; CHECK-NEXT: vsetivli zero, 6, e8, mf2, tu, ma +; CHECK-NEXT: vslideup.vi v0, v9, 4 ; CHECK-NEXT: lui a0, %hi(.LCPI10_6) ; CHECK-NEXT: addi a0, a0, %lo(.LCPI10_6) ; CHECK-NEXT: vsetivli zero, 16, e64, m8, ta, ma -; CHECK-NEXT: vle8.v v8, (a0) -; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vsaddu.vx v8, v16, a1 -; CHECK-NEXT: vmsltu.vx v16, v8, a2 -; CHECK-NEXT: vsetvli zero, zero, e8, m1, ta, ma -; CHECK-NEXT: vslideup.vi v0, v16, 14 +; CHECK-NEXT: vle8.v v9, (a0) +; CHECK-NEXT: vsext.vf8 v16, v9 +; CHECK-NEXT: vsaddu.vx v16, v16, a1 +; CHECK-NEXT: vmsltu.vx v9, v16, a2 +; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma +; CHECK-NEXT: vslideup.vi v0, v9, 6 +; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma +; CHECK-NEXT: vslideup.vi v0, v8, 8 ; CHECK-NEXT: ret %mask = call <128 x i1> @llvm.get.active.lane.mask.v128i1.i64(i64 %index, i64 %tc) ret <128 x i1> %mask diff --git a/llvm/test/CodeGen/RISCV/rvv/combine-store-extract-crash.ll b/llvm/test/CodeGen/RISCV/rvv/combine-store-extract-crash.ll index c64216180c2a..ed434deea1a8 100644 --- a/llvm/test/CodeGen/RISCV/rvv/combine-store-extract-crash.ll +++ b/llvm/test/CodeGen/RISCV/rvv/combine-store-extract-crash.ll @@ -19,7 +19,7 @@ define void @test(ptr %ref_array, ptr %sad_array) { ; RV32-NEXT: th.swia a0, (a1), 4, 0 ; RV32-NEXT: vsetivli zero, 4, e8, mf4, ta, ma ; RV32-NEXT: vle8.v v10, (a3) -; RV32-NEXT: vsetivli zero, 8, e8, m1, tu, ma +; RV32-NEXT: vsetivli zero, 8, e8, mf2, ta, ma ; RV32-NEXT: vslideup.vi v10, v9, 4 ; RV32-NEXT: vsetivli zero, 16, e32, m4, ta, ma ; RV32-NEXT: vzext.vf4 v12, v10 @@ -42,7 +42,7 @@ define void @test(ptr %ref_array, ptr %sad_array) { ; RV64-NEXT: th.swia a0, (a1), 4, 0 ; RV64-NEXT: vsetivli zero, 4, e8, mf4, ta, ma ; RV64-NEXT: vle8.v v10, (a3) -; RV64-NEXT: vsetivli zero, 8, e8, m1, tu, ma +; RV64-NEXT: vsetivli zero, 8, e8, mf2, ta, ma ; RV64-NEXT: vslideup.vi v10, v9, 4 ; RV64-NEXT: vsetivli zero, 16, e32, m4, ta, ma ; RV64-NEXT: vzext.vf4 v12, v10 diff --git a/llvm/test/CodeGen/RISCV/rvv/extract-subvector.ll b/llvm/test/CodeGen/RISCV/rvv/extract-subvector.ll index 76aa2b913c65..e15e6452163b 100644 --- a/llvm/test/CodeGen/RISCV/rvv/extract-subvector.ll +++ b/llvm/test/CodeGen/RISCV/rvv/extract-subvector.ll @@ -469,9 +469,8 @@ define @extract_nxv6f16_nxv12f16_6( %in) ; CHECK: # %bb.0: ; CHECK-NEXT: csrr a0, vlenb ; CHECK-NEXT: srli a0, a0, 2 -; CHECK-NEXT: vsetvli zero, a0, e16, m1, ta, ma -; CHECK-NEXT: vslidedown.vx v13, v10, a0 ; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma +; CHECK-NEXT: vslidedown.vx v13, v10, a0 ; CHECK-NEXT: vslidedown.vx v12, v9, a0 ; CHECK-NEXT: add a1, a0, a0 ; CHECK-NEXT: vsetvli zero, a1, e16, m1, ta, ma diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-concat.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-concat.ll index e5bef20fd9e2..98e6b8f2dd76 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-concat.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-concat.ll @@ -19,14 +19,11 @@ define <8 x i32> @concat_2xv4i32(<4 x i32> %a, <4 x i32> %b) { define <8 x i32> @concat_4xv2i32(<2 x i32> %a, <2 x i32> %b, <2 x i32> %c, <2 x i32> %d) { ; CHECK-LABEL: concat_4xv2i32: ; CHECK: # %bb.0: -; CHECK-NEXT: vmv1r.v v12, v11 -; CHECK-NEXT: vmv1r.v v14, v9 -; CHECK-NEXT: vsetivli zero, 4, e32, m2, tu, ma -; CHECK-NEXT: vslideup.vi v8, v14, 2 -; CHECK-NEXT: vsetivli zero, 6, e32, m2, tu, ma -; CHECK-NEXT: vslideup.vi v8, v10, 4 +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: vslideup.vi v10, v11, 2 +; CHECK-NEXT: vslideup.vi v8, v9, 2 ; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, ma -; CHECK-NEXT: vslideup.vi v8, v12, 6 +; CHECK-NEXT: vslideup.vi v8, v10, 4 ; CHECK-NEXT: ret %ab = shufflevector <2 x i32> %a, <2 x i32> %b, <4 x i32> %cd = shufflevector <2 x i32> %c, <2 x i32> %d, <4 x i32> @@ -37,24 +34,18 @@ define <8 x i32> @concat_4xv2i32(<2 x i32> %a, <2 x i32> %b, <2 x i32> %c, <2 x define <8 x i32> @concat_8xv1i32(<1 x i32> %a, <1 x i32> %b, <1 x i32> %c, <1 x i32> %d, <1 x i32> %e, <1 x i32> %f, <1 x i32> %g, <1 x i32> %h) { ; CHECK-LABEL: concat_8xv1i32: ; CHECK: # %bb.0: -; CHECK-NEXT: vmv1r.v v16, v15 -; CHECK-NEXT: vmv1r.v v18, v13 -; CHECK-NEXT: vmv1r.v v20, v11 -; CHECK-NEXT: vmv1r.v v22, v9 -; CHECK-NEXT: vsetivli zero, 2, e32, m2, tu, ma -; CHECK-NEXT: vslideup.vi v8, v22, 1 -; CHECK-NEXT: vsetivli zero, 3, e32, m2, tu, ma +; CHECK-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; CHECK-NEXT: vslideup.vi v14, v15, 1 +; CHECK-NEXT: vslideup.vi v12, v13, 1 +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: vslideup.vi v12, v14, 2 +; CHECK-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; CHECK-NEXT: vslideup.vi v10, v11, 1 +; CHECK-NEXT: vslideup.vi v8, v9, 1 +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma ; CHECK-NEXT: vslideup.vi v8, v10, 2 -; CHECK-NEXT: vsetivli zero, 4, e32, m2, tu, ma -; CHECK-NEXT: vslideup.vi v8, v20, 3 -; CHECK-NEXT: vsetivli zero, 5, e32, m2, tu, ma -; CHECK-NEXT: vslideup.vi v8, v12, 4 -; CHECK-NEXT: vsetivli zero, 6, e32, m2, tu, ma -; CHECK-NEXT: vslideup.vi v8, v18, 5 -; CHECK-NEXT: vsetivli zero, 7, e32, m2, tu, ma -; CHECK-NEXT: vslideup.vi v8, v14, 6 ; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, ma -; CHECK-NEXT: vslideup.vi v8, v16, 7 +; CHECK-NEXT: vslideup.vi v8, v12, 4 ; CHECK-NEXT: ret %ab = shufflevector <1 x i32> %a, <1 x i32> %b, <2 x i32> %cd = shufflevector <1 x i32> %c, <1 x i32> %d, <2 x i32> @@ -80,15 +71,14 @@ define <16 x i32> @concat_2xv8i32(<8 x i32> %a, <8 x i32> %b) { define <16 x i32> @concat_4xv4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c, <4 x i32> %d) { ; CHECK-LABEL: concat_4xv4i32: ; CHECK: # %bb.0: -; CHECK-NEXT: vmv1r.v v12, v11 -; CHECK-NEXT: vmv1r.v v16, v10 -; CHECK-NEXT: vmv1r.v v20, v9 -; CHECK-NEXT: vsetivli zero, 8, e32, m4, tu, ma -; CHECK-NEXT: vslideup.vi v8, v20, 4 -; CHECK-NEXT: vsetivli zero, 12, e32, m4, tu, ma -; CHECK-NEXT: vslideup.vi v8, v16, 8 +; CHECK-NEXT: vmv1r.v v14, v11 +; CHECK-NEXT: vmv1r.v v12, v10 +; CHECK-NEXT: vmv1r.v v10, v9 +; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-NEXT: vslideup.vi v12, v14, 4 +; CHECK-NEXT: vslideup.vi v8, v10, 4 ; CHECK-NEXT: vsetivli zero, 16, e32, m4, ta, ma -; CHECK-NEXT: vslideup.vi v8, v12, 12 +; CHECK-NEXT: vslideup.vi v8, v12, 8 ; CHECK-NEXT: ret %ab = shufflevector <4 x i32> %a, <4 x i32> %b, <8 x i32> %cd = shufflevector <4 x i32> %c, <4 x i32> %d, <8 x i32> @@ -99,26 +89,18 @@ define <16 x i32> @concat_4xv4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c, <4 x define <16 x i32> @concat_8xv2i32(<2 x i32> %a, <2 x i32> %b, <2 x i32> %c, <2 x i32> %d, <2 x i32> %e, <2 x i32> %f, <2 x i32> %g, <2 x i32> %h) { ; CHECK-LABEL: concat_8xv2i32: ; CHECK: # %bb.0: -; CHECK-NEXT: vmv1r.v v16, v15 -; CHECK-NEXT: vmv1r.v v20, v14 -; CHECK-NEXT: vmv1r.v v24, v13 -; CHECK-NEXT: vmv1r.v v28, v11 -; CHECK-NEXT: vmv1r.v v4, v10 -; CHECK-NEXT: vmv1r.v v0, v9 -; CHECK-NEXT: vsetivli zero, 4, e32, m4, tu, ma -; CHECK-NEXT: vslideup.vi v8, v0, 2 -; CHECK-NEXT: vsetivli zero, 6, e32, m4, tu, ma -; CHECK-NEXT: vslideup.vi v8, v4, 4 -; CHECK-NEXT: vsetivli zero, 8, e32, m4, tu, ma -; CHECK-NEXT: vslideup.vi v8, v28, 6 -; CHECK-NEXT: vsetivli zero, 10, e32, m4, tu, ma -; CHECK-NEXT: vslideup.vi v8, v12, 8 -; CHECK-NEXT: vsetivli zero, 12, e32, m4, tu, ma -; CHECK-NEXT: vslideup.vi v8, v24, 10 -; CHECK-NEXT: vsetivli zero, 14, e32, m4, tu, ma -; CHECK-NEXT: vslideup.vi v8, v20, 12 +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: vslideup.vi v14, v15, 2 +; CHECK-NEXT: vslideup.vi v12, v13, 2 +; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-NEXT: vslideup.vi v12, v14, 4 +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: vslideup.vi v10, v11, 2 +; CHECK-NEXT: vslideup.vi v8, v9, 2 +; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-NEXT: vslideup.vi v8, v10, 4 ; CHECK-NEXT: vsetivli zero, 16, e32, m4, ta, ma -; CHECK-NEXT: vslideup.vi v8, v16, 14 +; CHECK-NEXT: vslideup.vi v8, v12, 8 ; CHECK-NEXT: ret %ab = shufflevector <2 x i32> %a, <2 x i32> %b, <4 x i32> %cd = shufflevector <2 x i32> %c, <2 x i32> %d, <4 x i32> @@ -152,29 +134,27 @@ define <32 x i32> @concat_2xv16i32(<16 x i32> %a, <16 x i32> %b) { define <32 x i32> @concat_4xv8i32(<8 x i32> %a, <8 x i32> %b, <8 x i32> %c, <8 x i32> %d) { ; VLA-LABEL: concat_4xv8i32: ; VLA: # %bb.0: -; VLA-NEXT: vmv2r.v v16, v14 -; VLA-NEXT: vmv2r.v v24, v12 -; VLA-NEXT: vmv2r.v v0, v10 -; VLA-NEXT: vsetivli zero, 16, e32, m8, tu, ma -; VLA-NEXT: vslideup.vi v8, v0, 8 -; VLA-NEXT: vsetivli zero, 24, e32, m8, tu, ma -; VLA-NEXT: vslideup.vi v8, v24, 16 +; VLA-NEXT: vmv2r.v v20, v14 +; VLA-NEXT: vmv2r.v v16, v12 +; VLA-NEXT: vmv2r.v v12, v10 +; VLA-NEXT: vsetivli zero, 16, e32, m4, ta, ma +; VLA-NEXT: vslideup.vi v16, v20, 8 +; VLA-NEXT: vslideup.vi v8, v12, 8 ; VLA-NEXT: li a0, 32 ; VLA-NEXT: vsetvli zero, a0, e32, m8, ta, ma -; VLA-NEXT: vslideup.vi v8, v16, 24 +; VLA-NEXT: vslideup.vi v8, v16, 16 ; VLA-NEXT: ret ; ; VLS-LABEL: concat_4xv8i32: ; VLS: # %bb.0: -; VLS-NEXT: vmv2r.v v16, v14 -; VLS-NEXT: vmv2r.v v24, v12 -; VLS-NEXT: vmv2r.v v0, v10 -; VLS-NEXT: vsetivli zero, 16, e32, m8, tu, ma -; VLS-NEXT: vslideup.vi v8, v0, 8 -; VLS-NEXT: vsetivli zero, 24, e32, m8, tu, ma -; VLS-NEXT: vslideup.vi v8, v24, 16 +; VLS-NEXT: vmv2r.v v20, v14 +; VLS-NEXT: vmv2r.v v16, v12 +; VLS-NEXT: vmv2r.v v12, v10 +; VLS-NEXT: vsetivli zero, 16, e32, m4, ta, ma +; VLS-NEXT: vslideup.vi v16, v20, 8 +; VLS-NEXT: vslideup.vi v8, v12, 8 ; VLS-NEXT: vsetvli a0, zero, e32, m8, ta, ma -; VLS-NEXT: vslideup.vi v8, v16, 24 +; VLS-NEXT: vslideup.vi v8, v16, 16 ; VLS-NEXT: ret %ab = shufflevector <8 x i32> %a, <8 x i32> %b, <16 x i32> %cd = shufflevector <8 x i32> %c, <8 x i32> %d, <16 x i32> @@ -185,123 +165,49 @@ define <32 x i32> @concat_4xv8i32(<8 x i32> %a, <8 x i32> %b, <8 x i32> %c, <8 x define <32 x i32> @concat_8xv4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c, <4 x i32> %d, <4 x i32> %e, <4 x i32> %f, <4 x i32> %g, <4 x i32> %h) { ; VLA-LABEL: concat_8xv4i32: ; VLA: # %bb.0: -; VLA-NEXT: addi sp, sp, -16 -; VLA-NEXT: .cfi_def_cfa_offset 16 -; VLA-NEXT: csrr a0, vlenb -; VLA-NEXT: slli a0, a0, 5 -; VLA-NEXT: sub sp, sp, a0 -; VLA-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 32 * vlenb -; VLA-NEXT: vmv1r.v v16, v15 -; VLA-NEXT: csrr a0, vlenb -; VLA-NEXT: slli a0, a0, 3 -; VLA-NEXT: mv a1, a0 -; VLA-NEXT: slli a0, a0, 1 -; VLA-NEXT: add a0, a0, a1 -; VLA-NEXT: add a0, sp, a0 -; VLA-NEXT: addi a0, a0, 16 -; VLA-NEXT: vs8r.v v16, (a0) # Unknown-size Folded Spill -; VLA-NEXT: vmv1r.v v16, v14 -; VLA-NEXT: csrr a0, vlenb -; VLA-NEXT: slli a0, a0, 4 -; VLA-NEXT: add a0, sp, a0 -; VLA-NEXT: addi a0, a0, 16 -; VLA-NEXT: vs8r.v v16, (a0) # Unknown-size Folded Spill -; VLA-NEXT: vmv1r.v v16, v13 -; VLA-NEXT: csrr a0, vlenb -; VLA-NEXT: slli a0, a0, 3 -; VLA-NEXT: add a0, sp, a0 -; VLA-NEXT: addi a0, a0, 16 -; VLA-NEXT: vs8r.v v16, (a0) # Unknown-size Folded Spill +; VLA-NEXT: vmv1r.v v18, v15 +; VLA-NEXT: vmv1r.v v20, v14 +; VLA-NEXT: vmv1r.v v22, v13 ; VLA-NEXT: vmv1r.v v16, v12 -; VLA-NEXT: addi a0, sp, 16 -; VLA-NEXT: vs8r.v v16, (a0) # Unknown-size Folded Spill -; VLA-NEXT: vmv1r.v v0, v11 -; VLA-NEXT: vmv1r.v v24, v10 -; VLA-NEXT: vmv1r.v v16, v9 -; VLA-NEXT: vsetivli zero, 8, e32, m8, tu, ma -; VLA-NEXT: vslideup.vi v8, v16, 4 -; VLA-NEXT: vsetivli zero, 12, e32, m8, tu, ma -; VLA-NEXT: vslideup.vi v8, v24, 8 -; VLA-NEXT: vsetivli zero, 16, e32, m8, tu, ma -; VLA-NEXT: vslideup.vi v8, v0, 12 -; VLA-NEXT: vsetivli zero, 20, e32, m8, tu, ma -; VLA-NEXT: vl8r.v v16, (a0) # Unknown-size Folded Reload -; VLA-NEXT: vslideup.vi v8, v16, 16 -; VLA-NEXT: vsetivli zero, 24, e32, m8, tu, ma -; VLA-NEXT: csrr a0, vlenb -; VLA-NEXT: slli a0, a0, 3 -; VLA-NEXT: add a0, sp, a0 -; VLA-NEXT: addi a0, a0, 16 -; VLA-NEXT: vl8r.v v16, (a0) # Unknown-size Folded Reload -; VLA-NEXT: vslideup.vi v8, v16, 20 -; VLA-NEXT: vsetivli zero, 28, e32, m8, tu, ma -; VLA-NEXT: csrr a0, vlenb -; VLA-NEXT: slli a0, a0, 4 -; VLA-NEXT: add a0, sp, a0 -; VLA-NEXT: addi a0, a0, 16 -; VLA-NEXT: vl8r.v v16, (a0) # Unknown-size Folded Reload -; VLA-NEXT: vslideup.vi v8, v16, 24 +; VLA-NEXT: vmv1r.v v14, v11 +; VLA-NEXT: vmv1r.v v12, v10 +; VLA-NEXT: vmv1r.v v10, v9 +; VLA-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; VLA-NEXT: vslideup.vi v20, v18, 4 +; VLA-NEXT: vslideup.vi v16, v22, 4 +; VLA-NEXT: vsetivli zero, 16, e32, m4, ta, ma +; VLA-NEXT: vslideup.vi v16, v20, 8 +; VLA-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; VLA-NEXT: vslideup.vi v12, v14, 4 +; VLA-NEXT: vslideup.vi v8, v10, 4 +; VLA-NEXT: vsetivli zero, 16, e32, m4, ta, ma +; VLA-NEXT: vslideup.vi v8, v12, 8 ; VLA-NEXT: li a0, 32 ; VLA-NEXT: vsetvli zero, a0, e32, m8, ta, ma -; VLA-NEXT: csrr a0, vlenb -; VLA-NEXT: slli a0, a0, 3 -; VLA-NEXT: mv a1, a0 -; VLA-NEXT: slli a0, a0, 1 -; VLA-NEXT: add a0, a0, a1 -; VLA-NEXT: add a0, sp, a0 -; VLA-NEXT: addi a0, a0, 16 -; VLA-NEXT: vl8r.v v16, (a0) # Unknown-size Folded Reload -; VLA-NEXT: vslideup.vi v8, v16, 28 -; VLA-NEXT: csrr a0, vlenb -; VLA-NEXT: slli a0, a0, 5 -; VLA-NEXT: add sp, sp, a0 -; VLA-NEXT: addi sp, sp, 16 +; VLA-NEXT: vslideup.vi v8, v16, 16 ; VLA-NEXT: ret ; ; VLS-LABEL: concat_8xv4i32: ; VLS: # %bb.0: -; VLS-NEXT: addi sp, sp, -16 -; VLS-NEXT: .cfi_def_cfa_offset 16 -; VLS-NEXT: addi sp, sp, -512 -; VLS-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 32 * vlenb -; VLS-NEXT: vmv1r.v v16, v15 -; VLS-NEXT: addi a0, sp, 400 -; VLS-NEXT: vs8r.v v16, (a0) # Unknown-size Folded Spill -; VLS-NEXT: vmv1r.v v16, v14 -; VLS-NEXT: addi a0, sp, 272 -; VLS-NEXT: vs8r.v v16, (a0) # Unknown-size Folded Spill -; VLS-NEXT: vmv1r.v v16, v13 -; VLS-NEXT: addi a0, sp, 144 -; VLS-NEXT: vs8r.v v16, (a0) # Unknown-size Folded Spill +; VLS-NEXT: vmv1r.v v18, v15 +; VLS-NEXT: vmv1r.v v20, v14 +; VLS-NEXT: vmv1r.v v22, v13 ; VLS-NEXT: vmv1r.v v16, v12 -; VLS-NEXT: addi a0, sp, 16 -; VLS-NEXT: vs8r.v v16, (a0) # Unknown-size Folded Spill -; VLS-NEXT: vmv1r.v v0, v11 -; VLS-NEXT: vmv1r.v v24, v10 -; VLS-NEXT: vmv1r.v v16, v9 -; VLS-NEXT: vsetivli zero, 8, e32, m8, tu, ma -; VLS-NEXT: vslideup.vi v8, v16, 4 -; VLS-NEXT: vsetivli zero, 12, e32, m8, tu, ma -; VLS-NEXT: vslideup.vi v8, v24, 8 -; VLS-NEXT: vsetivli zero, 16, e32, m8, tu, ma -; VLS-NEXT: vslideup.vi v8, v0, 12 -; VLS-NEXT: vsetivli zero, 20, e32, m8, tu, ma -; VLS-NEXT: vl8r.v v16, (a0) # Unknown-size Folded Reload -; VLS-NEXT: vslideup.vi v8, v16, 16 -; VLS-NEXT: vsetivli zero, 24, e32, m8, tu, ma -; VLS-NEXT: addi a0, sp, 144 -; VLS-NEXT: vl8r.v v16, (a0) # Unknown-size Folded Reload -; VLS-NEXT: vslideup.vi v8, v16, 20 -; VLS-NEXT: vsetivli zero, 28, e32, m8, tu, ma -; VLS-NEXT: addi a0, sp, 272 -; VLS-NEXT: vl8r.v v16, (a0) # Unknown-size Folded Reload -; VLS-NEXT: vslideup.vi v8, v16, 24 +; VLS-NEXT: vmv1r.v v14, v11 +; VLS-NEXT: vmv1r.v v12, v10 +; VLS-NEXT: vmv1r.v v10, v9 +; VLS-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; VLS-NEXT: vslideup.vi v20, v18, 4 +; VLS-NEXT: vslideup.vi v16, v22, 4 +; VLS-NEXT: vsetivli zero, 16, e32, m4, ta, ma +; VLS-NEXT: vslideup.vi v16, v20, 8 +; VLS-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; VLS-NEXT: vslideup.vi v12, v14, 4 +; VLS-NEXT: vslideup.vi v8, v10, 4 +; VLS-NEXT: vsetivli zero, 16, e32, m4, ta, ma +; VLS-NEXT: vslideup.vi v8, v12, 8 ; VLS-NEXT: vsetvli a0, zero, e32, m8, ta, ma -; VLS-NEXT: addi a0, sp, 400 -; VLS-NEXT: vl8r.v v16, (a0) # Unknown-size Folded Reload -; VLS-NEXT: vslideup.vi v8, v16, 28 -; VLS-NEXT: addi sp, sp, 512 -; VLS-NEXT: addi sp, sp, 16 +; VLS-NEXT: vslideup.vi v8, v16, 16 ; VLS-NEXT: ret %ab = shufflevector <4 x i32> %a, <4 x i32> %b, <8 x i32> %cd = shufflevector <4 x i32> %c, <4 x i32> %d, <8 x i32> diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-combine.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-combine.ll index 4ec2e59672ad..657d52354aa3 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-combine.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-combine.ll @@ -27,13 +27,14 @@ define void @widen_3xv4i16(ptr %x, ptr %z) { ; CHECK-NEXT: vsetivli zero, 4, e16, mf2, ta, ma ; CHECK-NEXT: vle16.v v8, (a0) ; CHECK-NEXT: addi a2, a0, 8 -; CHECK-NEXT: vle16.v v10, (a2) +; CHECK-NEXT: vle16.v v9, (a2) ; CHECK-NEXT: addi a0, a0, 16 -; CHECK-NEXT: vle16.v v12, (a0) -; CHECK-NEXT: vsetivli zero, 8, e16, m2, tu, ma -; CHECK-NEXT: vslideup.vi v8, v10, 4 -; CHECK-NEXT: vsetivli zero, 12, e16, m2, tu, ma -; CHECK-NEXT: vslideup.vi v8, v12, 8 +; CHECK-NEXT: vle16.v v10, (a0) +; CHECK-NEXT: vsetivli zero, 8, e16, m1, ta, ma +; CHECK-NEXT: vslideup.vi v8, v9, 4 +; CHECK-NEXT: vsetivli zero, 16, e16, m2, ta, ma +; CHECK-NEXT: vslideup.vi v8, v10, 8 +; CHECK-NEXT: vsetivli zero, 12, e16, m2, ta, ma ; CHECK-NEXT: vse16.v v8, (a1) ; CHECK-NEXT: ret %a = load <4 x i16>, ptr %x @@ -72,20 +73,18 @@ define void @widen_4xv4i16(ptr %x, ptr %z) { define void @widen_4xv4i16_unaligned(ptr %x, ptr %z) { ; CHECK-NO-MISALIGN-LABEL: widen_4xv4i16_unaligned: ; CHECK-NO-MISALIGN: # %bb.0: -; CHECK-NO-MISALIGN-NEXT: vsetivli zero, 8, e8, mf2, ta, ma +; CHECK-NO-MISALIGN-NEXT: vsetivli zero, 8, e16, m1, ta, ma ; CHECK-NO-MISALIGN-NEXT: vle8.v v8, (a0) -; CHECK-NO-MISALIGN-NEXT: addi a2, a0, 8 -; CHECK-NO-MISALIGN-NEXT: vle8.v v10, (a2) ; CHECK-NO-MISALIGN-NEXT: addi a2, a0, 16 -; CHECK-NO-MISALIGN-NEXT: vle8.v v12, (a2) +; CHECK-NO-MISALIGN-NEXT: vle8.v v10, (a2) +; CHECK-NO-MISALIGN-NEXT: addi a2, a0, 8 ; CHECK-NO-MISALIGN-NEXT: addi a0, a0, 24 -; CHECK-NO-MISALIGN-NEXT: vle8.v v14, (a0) -; CHECK-NO-MISALIGN-NEXT: vsetivli zero, 8, e16, m2, tu, ma -; CHECK-NO-MISALIGN-NEXT: vslideup.vi v8, v10, 4 -; CHECK-NO-MISALIGN-NEXT: vsetivli zero, 12, e16, m2, tu, ma -; CHECK-NO-MISALIGN-NEXT: vslideup.vi v8, v12, 8 +; CHECK-NO-MISALIGN-NEXT: vle8.v v9, (a0) +; CHECK-NO-MISALIGN-NEXT: vle8.v v11, (a2) +; CHECK-NO-MISALIGN-NEXT: vslideup.vi v10, v9, 4 +; CHECK-NO-MISALIGN-NEXT: vslideup.vi v8, v11, 4 ; CHECK-NO-MISALIGN-NEXT: vsetivli zero, 16, e16, m2, ta, ma -; CHECK-NO-MISALIGN-NEXT: vslideup.vi v8, v14, 12 +; CHECK-NO-MISALIGN-NEXT: vslideup.vi v8, v10, 8 ; CHECK-NO-MISALIGN-NEXT: vse16.v v8, (a1) ; CHECK-NO-MISALIGN-NEXT: ret ; @@ -187,18 +186,17 @@ define void @strided_constant_mismatch_4xv4i16(ptr %x, ptr %z) { ; CHECK: # %bb.0: ; CHECK-NEXT: vsetivli zero, 4, e16, mf2, ta, ma ; CHECK-NEXT: vle16.v v8, (a0) -; CHECK-NEXT: addi a2, a0, 2 -; CHECK-NEXT: vle16.v v10, (a2) ; CHECK-NEXT: addi a2, a0, 6 -; CHECK-NEXT: vle16.v v12, (a2) +; CHECK-NEXT: vle16.v v10, (a2) +; CHECK-NEXT: addi a2, a0, 2 ; CHECK-NEXT: addi a0, a0, 8 -; CHECK-NEXT: vle16.v v14, (a0) -; CHECK-NEXT: vsetivli zero, 8, e16, m2, tu, ma -; CHECK-NEXT: vslideup.vi v8, v10, 4 -; CHECK-NEXT: vsetivli zero, 12, e16, m2, tu, ma -; CHECK-NEXT: vslideup.vi v8, v12, 8 +; CHECK-NEXT: vle16.v v9, (a0) +; CHECK-NEXT: vle16.v v11, (a2) +; CHECK-NEXT: vsetivli zero, 8, e16, m1, ta, ma +; CHECK-NEXT: vslideup.vi v10, v9, 4 +; CHECK-NEXT: vslideup.vi v8, v11, 4 ; CHECK-NEXT: vsetivli zero, 16, e16, m2, ta, ma -; CHECK-NEXT: vslideup.vi v8, v14, 12 +; CHECK-NEXT: vslideup.vi v8, v10, 8 ; CHECK-NEXT: vse16.v v8, (a1) ; CHECK-NEXT: ret %a = load <4 x i16>, ptr %x @@ -258,17 +256,16 @@ define void @strided_runtime_mismatch_4xv4i16(ptr %x, ptr %z, i64 %s, i64 %t) { ; RV32-NEXT: vsetivli zero, 4, e16, mf2, ta, ma ; RV32-NEXT: vle16.v v8, (a0) ; RV32-NEXT: add a0, a0, a2 -; RV32-NEXT: vle16.v v10, (a0) -; RV32-NEXT: add a0, a0, a4 -; RV32-NEXT: vle16.v v12, (a0) -; RV32-NEXT: add a0, a0, a2 -; RV32-NEXT: vle16.v v14, (a0) -; RV32-NEXT: vsetivli zero, 8, e16, m2, tu, ma -; RV32-NEXT: vslideup.vi v8, v10, 4 -; RV32-NEXT: vsetivli zero, 12, e16, m2, tu, ma -; RV32-NEXT: vslideup.vi v8, v12, 8 +; RV32-NEXT: add a4, a0, a4 +; RV32-NEXT: vle16.v v10, (a4) +; RV32-NEXT: add a2, a4, a2 +; RV32-NEXT: vle16.v v9, (a2) +; RV32-NEXT: vle16.v v11, (a0) +; RV32-NEXT: vsetivli zero, 8, e16, m1, ta, ma +; RV32-NEXT: vslideup.vi v10, v9, 4 +; RV32-NEXT: vslideup.vi v8, v11, 4 ; RV32-NEXT: vsetivli zero, 16, e16, m2, ta, ma -; RV32-NEXT: vslideup.vi v8, v14, 12 +; RV32-NEXT: vslideup.vi v8, v10, 8 ; RV32-NEXT: vse16.v v8, (a1) ; RV32-NEXT: ret ; @@ -277,17 +274,16 @@ define void @strided_runtime_mismatch_4xv4i16(ptr %x, ptr %z, i64 %s, i64 %t) { ; RV64-NEXT: vsetivli zero, 4, e16, mf2, ta, ma ; RV64-NEXT: vle16.v v8, (a0) ; RV64-NEXT: add a0, a0, a2 -; RV64-NEXT: vle16.v v10, (a0) -; RV64-NEXT: add a0, a0, a3 -; RV64-NEXT: vle16.v v12, (a0) -; RV64-NEXT: add a0, a0, a2 -; RV64-NEXT: vle16.v v14, (a0) -; RV64-NEXT: vsetivli zero, 8, e16, m2, tu, ma -; RV64-NEXT: vslideup.vi v8, v10, 4 -; RV64-NEXT: vsetivli zero, 12, e16, m2, tu, ma -; RV64-NEXT: vslideup.vi v8, v12, 8 +; RV64-NEXT: add a3, a0, a3 +; RV64-NEXT: vle16.v v10, (a3) +; RV64-NEXT: add a2, a3, a2 +; RV64-NEXT: vle16.v v9, (a2) +; RV64-NEXT: vle16.v v11, (a0) +; RV64-NEXT: vsetivli zero, 8, e16, m1, ta, ma +; RV64-NEXT: vslideup.vi v10, v9, 4 +; RV64-NEXT: vslideup.vi v8, v11, 4 ; RV64-NEXT: vsetivli zero, 16, e16, m2, ta, ma -; RV64-NEXT: vslideup.vi v8, v14, 12 +; RV64-NEXT: vslideup.vi v8, v10, 8 ; RV64-NEXT: vse16.v v8, (a1) ; RV64-NEXT: ret ; @@ -296,17 +292,16 @@ define void @strided_runtime_mismatch_4xv4i16(ptr %x, ptr %z, i64 %s, i64 %t) { ; ZVE64F-NEXT: vsetivli zero, 4, e16, mf2, ta, ma ; ZVE64F-NEXT: vle16.v v8, (a0) ; ZVE64F-NEXT: add a0, a0, a2 -; ZVE64F-NEXT: vle16.v v10, (a0) -; ZVE64F-NEXT: add a0, a0, a3 -; ZVE64F-NEXT: vle16.v v12, (a0) -; ZVE64F-NEXT: add a0, a0, a2 -; ZVE64F-NEXT: vle16.v v14, (a0) -; ZVE64F-NEXT: vsetivli zero, 8, e16, m2, tu, ma -; ZVE64F-NEXT: vslideup.vi v8, v10, 4 -; ZVE64F-NEXT: vsetivli zero, 12, e16, m2, tu, ma -; ZVE64F-NEXT: vslideup.vi v8, v12, 8 +; ZVE64F-NEXT: add a3, a0, a3 +; ZVE64F-NEXT: vle16.v v10, (a3) +; ZVE64F-NEXT: add a2, a3, a2 +; ZVE64F-NEXT: vle16.v v9, (a2) +; ZVE64F-NEXT: vle16.v v11, (a0) +; ZVE64F-NEXT: vsetivli zero, 8, e16, m1, ta, ma +; ZVE64F-NEXT: vslideup.vi v10, v9, 4 +; ZVE64F-NEXT: vslideup.vi v8, v11, 4 ; ZVE64F-NEXT: vsetivli zero, 16, e16, m2, ta, ma -; ZVE64F-NEXT: vslideup.vi v8, v14, 12 +; ZVE64F-NEXT: vslideup.vi v8, v10, 8 ; ZVE64F-NEXT: vse16.v v8, (a1) ; ZVE64F-NEXT: ret %a = load <4 x i16>, ptr %x diff --git a/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll b/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll index eb7894ede046..b3bda5973eb8 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll @@ -441,57 +441,50 @@ define <4 x i32> @stest_f16i32(<4 x half> %x) { ; CHECK-V-NEXT: slli a1, a1, 2 ; CHECK-V-NEXT: sub sp, sp, a1 ; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 4 * vlenb -; CHECK-V-NEXT: lhu s0, 24(a0) -; CHECK-V-NEXT: lhu s1, 16(a0) -; CHECK-V-NEXT: lhu s2, 0(a0) -; CHECK-V-NEXT: lhu a0, 8(a0) +; CHECK-V-NEXT: lhu s0, 0(a0) +; CHECK-V-NEXT: lhu s1, 8(a0) +; CHECK-V-NEXT: lhu s2, 16(a0) +; CHECK-V-NEXT: lhu a0, 24(a0) ; CHECK-V-NEXT: fmv.w.x fa0, a0 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz ; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s2 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s2 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 2, e64, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v8, v10, 1 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 ; CHECK-V-NEXT: add a0, sp, a0 ; CHECK-V-NEXT: addi a0, a0, 16 ; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s1 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 3, e64, m2, tu, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 -; CHECK-V-NEXT: add a0, sp, a0 -; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 2 -; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 -; CHECK-V-NEXT: add a0, sp, a0 -; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma ; CHECK-V-NEXT: fmv.w.x fa0, s0 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz +; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma +; CHECK-V-NEXT: vmv.s.x v10, a0 +; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 1 ; CHECK-V-NEXT: vsetivli zero, 4, e64, m2, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 ; CHECK-V-NEXT: add a0, sp, a0 ; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 3 +; CHECK-V-NEXT: vl2r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 2 ; CHECK-V-NEXT: vsetvli zero, zero, e32, m1, ta, ma ; CHECK-V-NEXT: vnclip.wi v8, v10, 0 ; CHECK-V-NEXT: csrr a0, vlenb @@ -609,57 +602,50 @@ define <4 x i32> @utesth_f16i32(<4 x half> %x) { ; CHECK-V-NEXT: slli a1, a1, 2 ; CHECK-V-NEXT: sub sp, sp, a1 ; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 4 * vlenb -; CHECK-V-NEXT: lhu s0, 24(a0) -; CHECK-V-NEXT: lhu s1, 16(a0) -; CHECK-V-NEXT: lhu s2, 0(a0) -; CHECK-V-NEXT: lhu a0, 8(a0) +; CHECK-V-NEXT: lhu s0, 0(a0) +; CHECK-V-NEXT: lhu s1, 8(a0) +; CHECK-V-NEXT: lhu s2, 16(a0) +; CHECK-V-NEXT: lhu a0, 24(a0) ; CHECK-V-NEXT: fmv.w.x fa0, a0 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz ; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s2 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s2 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 2, e64, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v8, v10, 1 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 ; CHECK-V-NEXT: add a0, sp, a0 ; CHECK-V-NEXT: addi a0, a0, 16 ; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s1 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 3, e64, m2, tu, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 -; CHECK-V-NEXT: add a0, sp, a0 -; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 2 -; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 -; CHECK-V-NEXT: add a0, sp, a0 -; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma ; CHECK-V-NEXT: fmv.w.x fa0, s0 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz +; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma +; CHECK-V-NEXT: vmv.s.x v10, a0 +; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 1 ; CHECK-V-NEXT: vsetivli zero, 4, e64, m2, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 ; CHECK-V-NEXT: add a0, sp, a0 ; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 3 +; CHECK-V-NEXT: vl2r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 2 ; CHECK-V-NEXT: vsetvli zero, zero, e32, m1, ta, ma ; CHECK-V-NEXT: vnclipu.wi v8, v10, 0 ; CHECK-V-NEXT: csrr a0, vlenb @@ -787,60 +773,53 @@ define <4 x i32> @ustest_f16i32(<4 x half> %x) { ; CHECK-V-NEXT: slli a1, a1, 2 ; CHECK-V-NEXT: sub sp, sp, a1 ; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 4 * vlenb -; CHECK-V-NEXT: lhu s0, 24(a0) -; CHECK-V-NEXT: lhu s1, 16(a0) -; CHECK-V-NEXT: lhu s2, 0(a0) -; CHECK-V-NEXT: lhu a0, 8(a0) +; CHECK-V-NEXT: lhu s0, 0(a0) +; CHECK-V-NEXT: lhu s1, 8(a0) +; CHECK-V-NEXT: lhu s2, 16(a0) +; CHECK-V-NEXT: lhu a0, 24(a0) ; CHECK-V-NEXT: fmv.w.x fa0, a0 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz ; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s2 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s2 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 2, e64, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v8, v10, 1 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 ; CHECK-V-NEXT: add a0, sp, a0 ; CHECK-V-NEXT: addi a0, a0, 16 ; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s1 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 3, e64, m2, tu, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 -; CHECK-V-NEXT: add a0, sp, a0 -; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 2 -; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 -; CHECK-V-NEXT: add a0, sp, a0 -; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma ; CHECK-V-NEXT: fmv.w.x fa0, s0 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 4, e64, m2, ta, ma +; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e64, m2, ta, ma ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 ; CHECK-V-NEXT: add a0, sp, a0 ; CHECK-V-NEXT: addi a0, a0, 16 ; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 3 +; CHECK-V-NEXT: vslideup.vi v8, v10, 2 ; CHECK-V-NEXT: li a0, -1 ; CHECK-V-NEXT: srli a0, a0, 32 -; CHECK-V-NEXT: vmin.vx v8, v10, a0 +; CHECK-V-NEXT: vmin.vx v8, v8, a0 ; CHECK-V-NEXT: vmax.vx v10, v8, zero ; CHECK-V-NEXT: vsetvli zero, zero, e32, m1, ta, ma ; CHECK-V-NEXT: vnsrl.wi v8, v10, 0 @@ -1404,90 +1383,125 @@ define <8 x i16> @stest_f16i16(<8 x half> %x) { ; CHECK-V-NEXT: .cfi_offset s5, -56 ; CHECK-V-NEXT: .cfi_offset s6, -64 ; CHECK-V-NEXT: csrr a1, vlenb -; CHECK-V-NEXT: slli a1, a1, 1 +; CHECK-V-NEXT: slli a1, a1, 2 ; CHECK-V-NEXT: sub sp, sp, a1 -; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xd0, 0x00, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 80 + 2 * vlenb -; CHECK-V-NEXT: lhu s0, 56(a0) -; CHECK-V-NEXT: lhu s1, 48(a0) -; CHECK-V-NEXT: lhu s2, 40(a0) -; CHECK-V-NEXT: lhu s3, 32(a0) -; CHECK-V-NEXT: lhu s4, 24(a0) -; CHECK-V-NEXT: lhu s5, 16(a0) -; CHECK-V-NEXT: lhu s6, 0(a0) -; CHECK-V-NEXT: lhu a0, 8(a0) +; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xd0, 0x00, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 80 + 4 * vlenb +; CHECK-V-NEXT: lhu s0, 0(a0) +; CHECK-V-NEXT: lhu s1, 8(a0) +; CHECK-V-NEXT: lhu s2, 16(a0) +; CHECK-V-NEXT: lhu s3, 24(a0) +; CHECK-V-NEXT: lhu s4, 32(a0) +; CHECK-V-NEXT: lhu s5, 40(a0) +; CHECK-V-NEXT: lhu s6, 48(a0) +; CHECK-V-NEXT: lhu a0, 56(a0) ; CHECK-V-NEXT: fmv.w.x fa0, a0 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz ; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s6 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 2, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v8, v10, 1 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s5 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 3, e32, m2, tu, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 2 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma ; CHECK-V-NEXT: fmv.w.x fa0, s4 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 4, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 3 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 2 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s3 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 5, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s2 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 4 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s2 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 6, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 5 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s1 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 7, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s0 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 6 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s0 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; CHECK-V-NEXT: vmv.s.x v10, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 7 +; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 2 +; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl2r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 4 ; CHECK-V-NEXT: vsetvli zero, zero, e16, m1, ta, ma ; CHECK-V-NEXT: vnclip.wi v8, v10, 0 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: slli a0, a0, 2 ; CHECK-V-NEXT: add sp, sp, a0 ; CHECK-V-NEXT: ld ra, 72(sp) # 8-byte Folded Reload ; CHECK-V-NEXT: ld s0, 64(sp) # 8-byte Folded Reload @@ -1682,90 +1696,125 @@ define <8 x i16> @utesth_f16i16(<8 x half> %x) { ; CHECK-V-NEXT: .cfi_offset s5, -56 ; CHECK-V-NEXT: .cfi_offset s6, -64 ; CHECK-V-NEXT: csrr a1, vlenb -; CHECK-V-NEXT: slli a1, a1, 1 +; CHECK-V-NEXT: slli a1, a1, 2 ; CHECK-V-NEXT: sub sp, sp, a1 -; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xd0, 0x00, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 80 + 2 * vlenb -; CHECK-V-NEXT: lhu s0, 56(a0) -; CHECK-V-NEXT: lhu s1, 48(a0) -; CHECK-V-NEXT: lhu s2, 40(a0) -; CHECK-V-NEXT: lhu s3, 32(a0) -; CHECK-V-NEXT: lhu s4, 24(a0) -; CHECK-V-NEXT: lhu s5, 16(a0) -; CHECK-V-NEXT: lhu s6, 0(a0) -; CHECK-V-NEXT: lhu a0, 8(a0) +; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xd0, 0x00, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 80 + 4 * vlenb +; CHECK-V-NEXT: lhu s0, 0(a0) +; CHECK-V-NEXT: lhu s1, 8(a0) +; CHECK-V-NEXT: lhu s2, 16(a0) +; CHECK-V-NEXT: lhu s3, 24(a0) +; CHECK-V-NEXT: lhu s4, 32(a0) +; CHECK-V-NEXT: lhu s5, 40(a0) +; CHECK-V-NEXT: lhu s6, 48(a0) +; CHECK-V-NEXT: lhu a0, 56(a0) ; CHECK-V-NEXT: fmv.w.x fa0, a0 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz ; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s6 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 2, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v8, v10, 1 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s5 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 3, e32, m2, tu, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 2 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma ; CHECK-V-NEXT: fmv.w.x fa0, s4 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 4, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 3 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 2 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s3 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 5, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s2 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 4 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s2 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 6, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 5 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s1 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 7, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s0 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 6 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s0 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; CHECK-V-NEXT: vmv.s.x v10, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 7 -; CHECK-V-NEXT: vsetvli zero, zero, e16, m1, ta, ma -; CHECK-V-NEXT: vnclipu.wi v8, v10, 0 +; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 2 +; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma ; CHECK-V-NEXT: csrr a0, vlenb ; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl2r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 4 +; CHECK-V-NEXT: vsetvli zero, zero, e16, m1, ta, ma +; CHECK-V-NEXT: vnclipu.wi v8, v10, 0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 2 ; CHECK-V-NEXT: add sp, sp, a0 ; CHECK-V-NEXT: ld ra, 72(sp) # 8-byte Folded Reload ; CHECK-V-NEXT: ld s0, 64(sp) # 8-byte Folded Reload @@ -1982,94 +2031,129 @@ define <8 x i16> @ustest_f16i16(<8 x half> %x) { ; CHECK-V-NEXT: .cfi_offset s5, -56 ; CHECK-V-NEXT: .cfi_offset s6, -64 ; CHECK-V-NEXT: csrr a1, vlenb -; CHECK-V-NEXT: slli a1, a1, 1 +; CHECK-V-NEXT: slli a1, a1, 2 ; CHECK-V-NEXT: sub sp, sp, a1 -; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xd0, 0x00, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 80 + 2 * vlenb -; CHECK-V-NEXT: lhu s0, 56(a0) -; CHECK-V-NEXT: lhu s1, 48(a0) -; CHECK-V-NEXT: lhu s2, 40(a0) -; CHECK-V-NEXT: lhu s3, 32(a0) -; CHECK-V-NEXT: lhu s4, 24(a0) -; CHECK-V-NEXT: lhu s5, 16(a0) -; CHECK-V-NEXT: lhu s6, 0(a0) -; CHECK-V-NEXT: lhu a0, 8(a0) +; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xd0, 0x00, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 80 + 4 * vlenb +; CHECK-V-NEXT: lhu s0, 0(a0) +; CHECK-V-NEXT: lhu s1, 8(a0) +; CHECK-V-NEXT: lhu s2, 16(a0) +; CHECK-V-NEXT: lhu s3, 24(a0) +; CHECK-V-NEXT: lhu s4, 32(a0) +; CHECK-V-NEXT: lhu s5, 40(a0) +; CHECK-V-NEXT: lhu s6, 48(a0) +; CHECK-V-NEXT: lhu a0, 56(a0) ; CHECK-V-NEXT: fmv.w.x fa0, a0 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz ; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s6 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 2, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v8, v10, 1 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s5 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 3, e32, m2, tu, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 2 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma ; CHECK-V-NEXT: fmv.w.x fa0, s4 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 4, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 3 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 2 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s3 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 5, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s2 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 4 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s2 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 6, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 5 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s1 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 7, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s0 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 6 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s0 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 2 +; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 ; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 7 +; CHECK-V-NEXT: vslideup.vi v8, v10, 4 ; CHECK-V-NEXT: lui a0, 16 ; CHECK-V-NEXT: addi a0, a0, -1 -; CHECK-V-NEXT: vmin.vx v8, v10, a0 +; CHECK-V-NEXT: vmin.vx v8, v8, a0 ; CHECK-V-NEXT: vmax.vx v10, v8, zero ; CHECK-V-NEXT: vsetvli zero, zero, e16, m1, ta, ma ; CHECK-V-NEXT: vnsrl.wi v8, v10, 0 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: slli a0, a0, 2 ; CHECK-V-NEXT: add sp, sp, a0 ; CHECK-V-NEXT: ld ra, 72(sp) # 8-byte Folded Reload ; CHECK-V-NEXT: ld s0, 64(sp) # 8-byte Folded Reload @@ -3723,57 +3807,50 @@ define <4 x i32> @stest_f16i32_mm(<4 x half> %x) { ; CHECK-V-NEXT: slli a1, a1, 2 ; CHECK-V-NEXT: sub sp, sp, a1 ; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 4 * vlenb -; CHECK-V-NEXT: lhu s0, 24(a0) -; CHECK-V-NEXT: lhu s1, 16(a0) -; CHECK-V-NEXT: lhu s2, 0(a0) -; CHECK-V-NEXT: lhu a0, 8(a0) +; CHECK-V-NEXT: lhu s0, 0(a0) +; CHECK-V-NEXT: lhu s1, 8(a0) +; CHECK-V-NEXT: lhu s2, 16(a0) +; CHECK-V-NEXT: lhu a0, 24(a0) ; CHECK-V-NEXT: fmv.w.x fa0, a0 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz ; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s2 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s2 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 2, e64, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v8, v10, 1 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 ; CHECK-V-NEXT: add a0, sp, a0 ; CHECK-V-NEXT: addi a0, a0, 16 ; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s1 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 3, e64, m2, tu, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 -; CHECK-V-NEXT: add a0, sp, a0 -; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 2 -; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 -; CHECK-V-NEXT: add a0, sp, a0 -; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma ; CHECK-V-NEXT: fmv.w.x fa0, s0 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz +; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma +; CHECK-V-NEXT: vmv.s.x v10, a0 +; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 1 ; CHECK-V-NEXT: vsetivli zero, 4, e64, m2, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 ; CHECK-V-NEXT: add a0, sp, a0 ; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 3 +; CHECK-V-NEXT: vl2r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 2 ; CHECK-V-NEXT: vsetvli zero, zero, e32, m1, ta, ma ; CHECK-V-NEXT: vnclip.wi v8, v10, 0 ; CHECK-V-NEXT: csrr a0, vlenb @@ -3889,57 +3966,50 @@ define <4 x i32> @utesth_f16i32_mm(<4 x half> %x) { ; CHECK-V-NEXT: slli a1, a1, 2 ; CHECK-V-NEXT: sub sp, sp, a1 ; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 4 * vlenb -; CHECK-V-NEXT: lhu s0, 24(a0) -; CHECK-V-NEXT: lhu s1, 16(a0) -; CHECK-V-NEXT: lhu s2, 0(a0) -; CHECK-V-NEXT: lhu a0, 8(a0) +; CHECK-V-NEXT: lhu s0, 0(a0) +; CHECK-V-NEXT: lhu s1, 8(a0) +; CHECK-V-NEXT: lhu s2, 16(a0) +; CHECK-V-NEXT: lhu a0, 24(a0) ; CHECK-V-NEXT: fmv.w.x fa0, a0 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz ; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s2 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s2 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 2, e64, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v8, v10, 1 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 ; CHECK-V-NEXT: add a0, sp, a0 ; CHECK-V-NEXT: addi a0, a0, 16 ; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s1 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 3, e64, m2, tu, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 -; CHECK-V-NEXT: add a0, sp, a0 -; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 2 -; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 -; CHECK-V-NEXT: add a0, sp, a0 -; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma ; CHECK-V-NEXT: fmv.w.x fa0, s0 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz +; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma +; CHECK-V-NEXT: vmv.s.x v10, a0 +; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 1 ; CHECK-V-NEXT: vsetivli zero, 4, e64, m2, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 ; CHECK-V-NEXT: add a0, sp, a0 ; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 3 +; CHECK-V-NEXT: vl2r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 2 ; CHECK-V-NEXT: vsetvli zero, zero, e32, m1, ta, ma ; CHECK-V-NEXT: vnclipu.wi v8, v10, 0 ; CHECK-V-NEXT: csrr a0, vlenb @@ -4066,60 +4136,53 @@ define <4 x i32> @ustest_f16i32_mm(<4 x half> %x) { ; CHECK-V-NEXT: slli a1, a1, 2 ; CHECK-V-NEXT: sub sp, sp, a1 ; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 4 * vlenb -; CHECK-V-NEXT: lhu s0, 24(a0) -; CHECK-V-NEXT: lhu s1, 16(a0) -; CHECK-V-NEXT: lhu s2, 0(a0) -; CHECK-V-NEXT: lhu a0, 8(a0) +; CHECK-V-NEXT: lhu s0, 0(a0) +; CHECK-V-NEXT: lhu s1, 8(a0) +; CHECK-V-NEXT: lhu s2, 16(a0) +; CHECK-V-NEXT: lhu a0, 24(a0) ; CHECK-V-NEXT: fmv.w.x fa0, a0 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz ; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s2 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s2 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 2, e64, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v8, v10, 1 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 ; CHECK-V-NEXT: add a0, sp, a0 ; CHECK-V-NEXT: addi a0, a0, 16 ; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s1 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 3, e64, m2, tu, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 -; CHECK-V-NEXT: add a0, sp, a0 -; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 2 -; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 -; CHECK-V-NEXT: add a0, sp, a0 -; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma ; CHECK-V-NEXT: fmv.w.x fa0, s0 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 4, e64, m2, ta, ma +; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e64, m2, ta, ma ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 ; CHECK-V-NEXT: add a0, sp, a0 ; CHECK-V-NEXT: addi a0, a0, 16 ; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 3 +; CHECK-V-NEXT: vslideup.vi v8, v10, 2 ; CHECK-V-NEXT: li a0, -1 ; CHECK-V-NEXT: srli a0, a0, 32 -; CHECK-V-NEXT: vmin.vx v8, v10, a0 +; CHECK-V-NEXT: vmin.vx v8, v8, a0 ; CHECK-V-NEXT: vmax.vx v10, v8, zero ; CHECK-V-NEXT: vsetvli zero, zero, e32, m1, ta, ma ; CHECK-V-NEXT: vnsrl.wi v8, v10, 0 @@ -4671,90 +4734,125 @@ define <8 x i16> @stest_f16i16_mm(<8 x half> %x) { ; CHECK-V-NEXT: .cfi_offset s5, -56 ; CHECK-V-NEXT: .cfi_offset s6, -64 ; CHECK-V-NEXT: csrr a1, vlenb -; CHECK-V-NEXT: slli a1, a1, 1 +; CHECK-V-NEXT: slli a1, a1, 2 ; CHECK-V-NEXT: sub sp, sp, a1 -; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xd0, 0x00, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 80 + 2 * vlenb -; CHECK-V-NEXT: lhu s0, 56(a0) -; CHECK-V-NEXT: lhu s1, 48(a0) -; CHECK-V-NEXT: lhu s2, 40(a0) -; CHECK-V-NEXT: lhu s3, 32(a0) -; CHECK-V-NEXT: lhu s4, 24(a0) -; CHECK-V-NEXT: lhu s5, 16(a0) -; CHECK-V-NEXT: lhu s6, 0(a0) -; CHECK-V-NEXT: lhu a0, 8(a0) +; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xd0, 0x00, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 80 + 4 * vlenb +; CHECK-V-NEXT: lhu s0, 0(a0) +; CHECK-V-NEXT: lhu s1, 8(a0) +; CHECK-V-NEXT: lhu s2, 16(a0) +; CHECK-V-NEXT: lhu s3, 24(a0) +; CHECK-V-NEXT: lhu s4, 32(a0) +; CHECK-V-NEXT: lhu s5, 40(a0) +; CHECK-V-NEXT: lhu s6, 48(a0) +; CHECK-V-NEXT: lhu a0, 56(a0) ; CHECK-V-NEXT: fmv.w.x fa0, a0 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz ; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s6 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 2, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v8, v10, 1 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s5 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 3, e32, m2, tu, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 2 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma ; CHECK-V-NEXT: fmv.w.x fa0, s4 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 4, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 3 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 2 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s3 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 5, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s2 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 4 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s2 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 6, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 5 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s1 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 7, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s0 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 6 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s0 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; CHECK-V-NEXT: vmv.s.x v10, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 7 +; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 2 +; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl2r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 4 ; CHECK-V-NEXT: vsetvli zero, zero, e16, m1, ta, ma ; CHECK-V-NEXT: vnclip.wi v8, v10, 0 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: slli a0, a0, 2 ; CHECK-V-NEXT: add sp, sp, a0 ; CHECK-V-NEXT: ld ra, 72(sp) # 8-byte Folded Reload ; CHECK-V-NEXT: ld s0, 64(sp) # 8-byte Folded Reload @@ -4947,90 +5045,125 @@ define <8 x i16> @utesth_f16i16_mm(<8 x half> %x) { ; CHECK-V-NEXT: .cfi_offset s5, -56 ; CHECK-V-NEXT: .cfi_offset s6, -64 ; CHECK-V-NEXT: csrr a1, vlenb -; CHECK-V-NEXT: slli a1, a1, 1 +; CHECK-V-NEXT: slli a1, a1, 2 ; CHECK-V-NEXT: sub sp, sp, a1 -; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xd0, 0x00, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 80 + 2 * vlenb -; CHECK-V-NEXT: lhu s0, 56(a0) -; CHECK-V-NEXT: lhu s1, 48(a0) -; CHECK-V-NEXT: lhu s2, 40(a0) -; CHECK-V-NEXT: lhu s3, 32(a0) -; CHECK-V-NEXT: lhu s4, 24(a0) -; CHECK-V-NEXT: lhu s5, 16(a0) -; CHECK-V-NEXT: lhu s6, 0(a0) -; CHECK-V-NEXT: lhu a0, 8(a0) +; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xd0, 0x00, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 80 + 4 * vlenb +; CHECK-V-NEXT: lhu s0, 0(a0) +; CHECK-V-NEXT: lhu s1, 8(a0) +; CHECK-V-NEXT: lhu s2, 16(a0) +; CHECK-V-NEXT: lhu s3, 24(a0) +; CHECK-V-NEXT: lhu s4, 32(a0) +; CHECK-V-NEXT: lhu s5, 40(a0) +; CHECK-V-NEXT: lhu s6, 48(a0) +; CHECK-V-NEXT: lhu a0, 56(a0) ; CHECK-V-NEXT: fmv.w.x fa0, a0 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz ; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s6 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 2, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v8, v10, 1 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s5 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 3, e32, m2, tu, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 2 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma ; CHECK-V-NEXT: fmv.w.x fa0, s4 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 4, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 3 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 2 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s3 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 5, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s2 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 4 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s2 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 6, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 5 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s1 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 7, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s0 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 6 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s0 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; CHECK-V-NEXT: vmv.s.x v10, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 7 +; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 2 +; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl2r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 4 ; CHECK-V-NEXT: vsetvli zero, zero, e16, m1, ta, ma ; CHECK-V-NEXT: vnclipu.wi v8, v10, 0 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: slli a0, a0, 2 ; CHECK-V-NEXT: add sp, sp, a0 ; CHECK-V-NEXT: ld ra, 72(sp) # 8-byte Folded Reload ; CHECK-V-NEXT: ld s0, 64(sp) # 8-byte Folded Reload @@ -5246,94 +5379,129 @@ define <8 x i16> @ustest_f16i16_mm(<8 x half> %x) { ; CHECK-V-NEXT: .cfi_offset s5, -56 ; CHECK-V-NEXT: .cfi_offset s6, -64 ; CHECK-V-NEXT: csrr a1, vlenb -; CHECK-V-NEXT: slli a1, a1, 1 +; CHECK-V-NEXT: slli a1, a1, 2 ; CHECK-V-NEXT: sub sp, sp, a1 -; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xd0, 0x00, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 80 + 2 * vlenb -; CHECK-V-NEXT: lhu s0, 56(a0) -; CHECK-V-NEXT: lhu s1, 48(a0) -; CHECK-V-NEXT: lhu s2, 40(a0) -; CHECK-V-NEXT: lhu s3, 32(a0) -; CHECK-V-NEXT: lhu s4, 24(a0) -; CHECK-V-NEXT: lhu s5, 16(a0) -; CHECK-V-NEXT: lhu s6, 0(a0) -; CHECK-V-NEXT: lhu a0, 8(a0) +; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xd0, 0x00, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 80 + 4 * vlenb +; CHECK-V-NEXT: lhu s0, 0(a0) +; CHECK-V-NEXT: lhu s1, 8(a0) +; CHECK-V-NEXT: lhu s2, 16(a0) +; CHECK-V-NEXT: lhu s3, 24(a0) +; CHECK-V-NEXT: lhu s4, 32(a0) +; CHECK-V-NEXT: lhu s5, 40(a0) +; CHECK-V-NEXT: lhu s6, 48(a0) +; CHECK-V-NEXT: lhu a0, 56(a0) ; CHECK-V-NEXT: fmv.w.x fa0, a0 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz ; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s6 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 2, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v8, v10, 1 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s5 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 3, e32, m2, tu, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 2 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma ; CHECK-V-NEXT: fmv.w.x fa0, s4 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 4, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 3 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 2 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s3 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 5, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s2 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 4 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s2 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 6, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 5 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s1 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 7, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s0 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 6 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s0 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 2 +; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 ; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 7 +; CHECK-V-NEXT: vslideup.vi v8, v10, 4 ; CHECK-V-NEXT: lui a0, 16 ; CHECK-V-NEXT: addi a0, a0, -1 -; CHECK-V-NEXT: vmin.vx v8, v10, a0 +; CHECK-V-NEXT: vmin.vx v8, v8, a0 ; CHECK-V-NEXT: vmax.vx v10, v8, zero ; CHECK-V-NEXT: vsetvli zero, zero, e16, m1, ta, ma ; CHECK-V-NEXT: vnsrl.wi v8, v10, 0 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: slli a0, a0, 2 ; CHECK-V-NEXT: add sp, sp, a0 ; CHECK-V-NEXT: ld ra, 72(sp) # 8-byte Folded Reload ; CHECK-V-NEXT: ld s0, 64(sp) # 8-byte Folded Reload diff --git a/llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll index f3ae03af7c78..0b236f6d3ff3 100644 --- a/llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll +++ b/llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll @@ -2136,17 +2136,18 @@ define @mgather_baseidx_nxv32i8(ptr %base, ; RV64-NEXT: vluxei64.v v13, (a0), v24, v0.t ; RV64-NEXT: srli a1, a1, 2 ; RV64-NEXT: vsetvli a3, zero, e8, mf2, ta, ma -; RV64-NEXT: vslidedown.vx v0, v16, a1 -; RV64-NEXT: vsetvli a1, zero, e64, m8, ta, ma -; RV64-NEXT: vsext.vf8 v16, v10 -; RV64-NEXT: vsetvli zero, zero, e8, m1, ta, mu -; RV64-NEXT: vluxei64.v v14, (a0), v16, v0.t +; RV64-NEXT: vslidedown.vx v8, v16, a1 ; RV64-NEXT: vsetvli a1, zero, e8, mf4, ta, ma -; RV64-NEXT: vslidedown.vx v0, v0, a2 +; RV64-NEXT: vslidedown.vx v0, v8, a2 ; RV64-NEXT: vsetvli a1, zero, e64, m8, ta, ma ; RV64-NEXT: vsext.vf8 v16, v11 ; RV64-NEXT: vsetvli zero, zero, e8, m1, ta, mu ; RV64-NEXT: vluxei64.v v15, (a0), v16, v0.t +; RV64-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; RV64-NEXT: vsext.vf8 v16, v10 +; RV64-NEXT: vsetvli zero, zero, e8, m1, ta, mu +; RV64-NEXT: vmv1r.v v0, v8 +; RV64-NEXT: vluxei64.v v14, (a0), v16, v0.t ; RV64-NEXT: vmv4r.v v8, v12 ; RV64-NEXT: ret %ptrs = getelementptr inbounds i8, ptr %base, %idxs diff --git a/llvm/test/CodeGen/RISCV/rvv/pr63596.ll b/llvm/test/CodeGen/RISCV/rvv/pr63596.ll index c27488b18a01..d13d67fd0a88 100644 --- a/llvm/test/CodeGen/RISCV/rvv/pr63596.ll +++ b/llvm/test/CodeGen/RISCV/rvv/pr63596.ll @@ -9,39 +9,38 @@ define <4 x float> @foo(ptr %0) nounwind { ; CHECK-NEXT: sd s0, 32(sp) # 8-byte Folded Spill ; CHECK-NEXT: sd s1, 24(sp) # 8-byte Folded Spill ; CHECK-NEXT: sd s2, 16(sp) # 8-byte Folded Spill -; CHECK-NEXT: lhu s0, 6(a0) -; CHECK-NEXT: lhu s1, 4(a0) -; CHECK-NEXT: lhu s2, 0(a0) -; CHECK-NEXT: lhu a0, 2(a0) +; CHECK-NEXT: lhu s0, 0(a0) +; CHECK-NEXT: lhu s1, 2(a0) +; CHECK-NEXT: lhu s2, 4(a0) +; CHECK-NEXT: lhu a0, 6(a0) ; CHECK-NEXT: fmv.w.x fa0, a0 ; CHECK-NEXT: call __extendhfsf2 -; CHECK-NEXT: fsw fa0, 8(sp) +; CHECK-NEXT: fsw fa0, 4(sp) ; CHECK-NEXT: fmv.w.x fa0, s2 ; CHECK-NEXT: call __extendhfsf2 -; CHECK-NEXT: fsw fa0, 0(sp) +; CHECK-NEXT: fsw fa0, 12(sp) ; CHECK-NEXT: fmv.w.x fa0, s1 ; CHECK-NEXT: call __extendhfsf2 -; CHECK-NEXT: fsw fa0, 12(sp) +; CHECK-NEXT: fsw fa0, 8(sp) ; CHECK-NEXT: fmv.w.x fa0, s0 ; CHECK-NEXT: call __extendhfsf2 -; CHECK-NEXT: fsw fa0, 4(sp) -; CHECK-NEXT: addi a0, sp, 8 +; CHECK-NEXT: fsw fa0, 0(sp) +; CHECK-NEXT: addi a0, sp, 4 ; CHECK-NEXT: vsetivli zero, 1, e32, mf2, ta, ma -; CHECK-NEXT: vle32.v v9, (a0) -; CHECK-NEXT: mv a0, sp ; CHECK-NEXT: vle32.v v8, (a0) -; CHECK-NEXT: vsetivli zero, 2, e32, m1, tu, ma -; CHECK-NEXT: vslideup.vi v8, v9, 1 ; CHECK-NEXT: addi a0, sp, 12 -; CHECK-NEXT: vsetivli zero, 1, e32, mf2, ta, ma ; CHECK-NEXT: vle32.v v9, (a0) -; CHECK-NEXT: vsetivli zero, 3, e32, m1, tu, ma -; CHECK-NEXT: vslideup.vi v8, v9, 2 -; CHECK-NEXT: addi a0, sp, 4 +; CHECK-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; CHECK-NEXT: vslideup.vi v9, v8, 1 +; CHECK-NEXT: addi a0, sp, 8 ; CHECK-NEXT: vsetivli zero, 1, e32, mf2, ta, ma -; CHECK-NEXT: vle32.v v9, (a0) +; CHECK-NEXT: vle32.v v10, (a0) +; CHECK-NEXT: mv a0, sp +; CHECK-NEXT: vle32.v v8, (a0) +; CHECK-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; CHECK-NEXT: vslideup.vi v8, v10, 1 ; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma -; CHECK-NEXT: vslideup.vi v8, v9, 3 +; CHECK-NEXT: vslideup.vi v8, v9, 2 ; CHECK-NEXT: ld ra, 40(sp) # 8-byte Folded Reload ; CHECK-NEXT: ld s0, 32(sp) # 8-byte Folded Reload ; CHECK-NEXT: ld s1, 24(sp) # 8-byte Folded Reload -- GitLab From 51d5b6581912c8495360a09a0e6be978e0374d90 Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Fri, 22 Mar 2024 07:26:29 +0800 Subject: [PATCH 010/306] [RISCV] Handle scalable ops with < EEW / 2 narrow types in combineBinOp_VLToVWBinOp_VL (#84158) We can remove the restriction that the narrow type needs to be exactly EEW / 2 for scalable ISD::{ADD,SUB,MUL} nodes. This allows us to perform the combine even if we can't fully fold the extend into the widening op. VP intrinsics already do this, since they are lowered to _VL nodes which don't have this restriction. The "exactly EEW / 2" narrow type restriction prevented us from emitting V{S,Z}EXT_VL nodes with i1 element types which crash when we try to select them, since no other legal type is double the size of i1, see the test case added in this PR `i1_zext`. So to preserve this, this adds a check for i1 narrow types instead. --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 8 +- .../RISCV/rvv/vscale-vw-web-simplification.ll | 38 +- llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll | 483 ++++++++++-------- llvm/test/CodeGen/RISCV/rvv/vwmul-sdnode.ll | 384 +++++++------- llvm/test/CodeGen/RISCV/rvv/vwsub-sdnode.ll | 320 ++++++------ 5 files changed, 631 insertions(+), 602 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 71059b5bdc0f..5a2fb0239e0a 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -13715,12 +13715,8 @@ struct NodeExtensionHelper { SDValue NarrowElt = OrigOperand.getOperand(0); MVT NarrowVT = NarrowElt.getSimpleValueType(); - - unsigned ScalarBits = VT.getScalarSizeInBits(); - unsigned NarrowScalarBits = NarrowVT.getScalarSizeInBits(); - - // Ensure the extension's semantic is equivalent to rvv vzext or vsext. - if (ScalarBits != NarrowScalarBits * 2) + // i1 types are legal but we can't select V{S,Z}EXT_VLs with them. + if (NarrowVT.getVectorElementType() == MVT::i1) break; SupportsZExt = Opc == ISD::ZERO_EXTEND; diff --git a/llvm/test/CodeGen/RISCV/rvv/vscale-vw-web-simplification.ll b/llvm/test/CodeGen/RISCV/rvv/vscale-vw-web-simplification.ll index 972fa66917a5..e56dca0732bb 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vscale-vw-web-simplification.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vscale-vw-web-simplification.ll @@ -283,18 +283,19 @@ define @vwop_vscale_sext_i8i32_multiple_users(ptr %x, ptr %y, ; ; FOLDING-LABEL: vwop_vscale_sext_i8i32_multiple_users: ; FOLDING: # %bb.0: -; FOLDING-NEXT: vsetvli a3, zero, e32, m1, ta, ma +; FOLDING-NEXT: vsetvli a3, zero, e16, mf2, ta, ma ; FOLDING-NEXT: vle8.v v8, (a0) ; FOLDING-NEXT: vle8.v v9, (a1) ; FOLDING-NEXT: vle8.v v10, (a2) -; FOLDING-NEXT: vsext.vf4 v11, v8 -; FOLDING-NEXT: vsext.vf4 v8, v9 -; FOLDING-NEXT: vsext.vf4 v9, v10 -; FOLDING-NEXT: vmul.vv v8, v11, v8 -; FOLDING-NEXT: vadd.vv v10, v11, v9 -; FOLDING-NEXT: vsub.vv v9, v11, v9 -; FOLDING-NEXT: vor.vv v8, v8, v10 -; FOLDING-NEXT: vor.vv v8, v8, v9 +; FOLDING-NEXT: vsext.vf2 v11, v8 +; FOLDING-NEXT: vsext.vf2 v8, v9 +; FOLDING-NEXT: vsext.vf2 v9, v10 +; FOLDING-NEXT: vwmul.vv v10, v11, v8 +; FOLDING-NEXT: vwadd.vv v8, v11, v9 +; FOLDING-NEXT: vwsub.vv v12, v11, v9 +; FOLDING-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; FOLDING-NEXT: vor.vv v8, v10, v8 +; FOLDING-NEXT: vor.vv v8, v8, v12 ; FOLDING-NEXT: ret %a = load , ptr %x %b = load , ptr %y @@ -563,18 +564,19 @@ define @vwop_vscale_zext_i8i32_multiple_users(ptr %x, ptr %y, ; ; FOLDING-LABEL: vwop_vscale_zext_i8i32_multiple_users: ; FOLDING: # %bb.0: -; FOLDING-NEXT: vsetvli a3, zero, e32, m1, ta, ma +; FOLDING-NEXT: vsetvli a3, zero, e16, mf2, ta, ma ; FOLDING-NEXT: vle8.v v8, (a0) ; FOLDING-NEXT: vle8.v v9, (a1) ; FOLDING-NEXT: vle8.v v10, (a2) -; FOLDING-NEXT: vzext.vf4 v11, v8 -; FOLDING-NEXT: vzext.vf4 v8, v9 -; FOLDING-NEXT: vzext.vf4 v9, v10 -; FOLDING-NEXT: vmul.vv v8, v11, v8 -; FOLDING-NEXT: vadd.vv v10, v11, v9 -; FOLDING-NEXT: vsub.vv v9, v11, v9 -; FOLDING-NEXT: vor.vv v8, v8, v10 -; FOLDING-NEXT: vor.vv v8, v8, v9 +; FOLDING-NEXT: vzext.vf2 v11, v8 +; FOLDING-NEXT: vzext.vf2 v8, v9 +; FOLDING-NEXT: vzext.vf2 v9, v10 +; FOLDING-NEXT: vwmulu.vv v10, v11, v8 +; FOLDING-NEXT: vwaddu.vv v8, v11, v9 +; FOLDING-NEXT: vwsubu.vv v12, v11, v9 +; FOLDING-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; FOLDING-NEXT: vor.vv v8, v10, v8 +; FOLDING-NEXT: vor.vv v8, v8, v12 ; FOLDING-NEXT: ret %a = load , ptr %x %b = load , ptr %y diff --git a/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll index a559fbf2bc8a..66a7eea18be5 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple=riscv32 -mattr=+v -verify-machineinstrs < %s | FileCheck %s -; RUN: llc -mtriple=riscv64 -mattr=+v -verify-machineinstrs < %s | FileCheck %s +; RUN: llc -mtriple=riscv32 -mattr=+v -verify-machineinstrs < %s | FileCheck -check-prefixes=CHECK,RV32 %s +; RUN: llc -mtriple=riscv64 -mattr=+v -verify-machineinstrs < %s | FileCheck -check-prefixes=CHECK,RV64 %s define @vwadd_vv_nxv1i64_nxv1i32( %va, %vb) { ; CHECK-LABEL: vwadd_vv_nxv1i64_nxv1i32: @@ -421,10 +421,10 @@ define @vwaddu_wx_nxv8i64_nxv8i32( %va, i32 define @vwadd_vv_nxv1i64_nxv1i16( %va, %vb) { ; CHECK-LABEL: vwadd_vv_nxv1i64_nxv1i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v8 -; CHECK-NEXT: vsext.vf4 v8, v9 -; CHECK-NEXT: vadd.vv v8, v10, v8 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v8 +; CHECK-NEXT: vsext.vf2 v11, v9 +; CHECK-NEXT: vwadd.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -435,10 +435,10 @@ define @vwadd_vv_nxv1i64_nxv1i16( %va, @vwaddu_vv_nxv1i64_nxv1i16( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv1i64_nxv1i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v8, v9 -; CHECK-NEXT: vadd.vv v8, v10, v8 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v8 +; CHECK-NEXT: vzext.vf2 v11, v9 +; CHECK-NEXT: vwaddu.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -451,10 +451,10 @@ define @vwadd_vx_nxv1i64_nxv1i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v8 -; CHECK-NEXT: vsext.vf4 v8, v9 -; CHECK-NEXT: vadd.vv v8, v10, v8 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v8 +; CHECK-NEXT: vsext.vf2 v11, v9 +; CHECK-NEXT: vwadd.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -469,10 +469,10 @@ define @vwaddu_vx_nxv1i64_nxv1i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v8, v9 -; CHECK-NEXT: vadd.vv v8, v10, v8 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v8 +; CHECK-NEXT: vzext.vf2 v11, v9 +; CHECK-NEXT: vwaddu.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -485,9 +485,9 @@ define @vwaddu_vx_nxv1i64_nxv1i16( %va, i16 define @vwadd_wv_nxv1i64_nxv1i16( %va, %vb) { ; CHECK-LABEL: vwadd_wv_nxv1i64_nxv1i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v9 -; CHECK-NEXT: vadd.vv v8, v8, v10 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v9 +; CHECK-NEXT: vwadd.wv v8, v8, v10 ; CHECK-NEXT: ret %vc = sext %vb to %vd = add %va, %vc @@ -497,9 +497,9 @@ define @vwadd_wv_nxv1i64_nxv1i16( %va, @vwaddu_wv_nxv1i64_nxv1i16( %va, %vb) { ; CHECK-LABEL: vwaddu_wv_nxv1i64_nxv1i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v9 -; CHECK-NEXT: vadd.vv v8, v8, v10 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v9 +; CHECK-NEXT: vwaddu.wv v8, v8, v10 ; CHECK-NEXT: ret %vc = zext %vb to %vd = add %va, %vc @@ -511,9 +511,9 @@ define @vwadd_wx_nxv1i64_nxv1i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v9 -; CHECK-NEXT: vadd.vv v8, v8, v10 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v9 +; CHECK-NEXT: vwadd.wv v8, v8, v10 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -527,9 +527,9 @@ define @vwaddu_wx_nxv1i64_nxv1i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v9 -; CHECK-NEXT: vadd.vv v8, v8, v10 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v9 +; CHECK-NEXT: vwaddu.wv v8, v8, v10 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -541,10 +541,10 @@ define @vwaddu_wx_nxv1i64_nxv1i16( %va, i16 define @vwadd_vv_nxv2i64_nxv2i16( %va, %vb) { ; CHECK-LABEL: vwadd_vv_nxv2i64_nxv2i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v8 -; CHECK-NEXT: vsext.vf4 v12, v9 -; CHECK-NEXT: vadd.vv v8, v10, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v8 +; CHECK-NEXT: vsext.vf2 v11, v9 +; CHECK-NEXT: vwadd.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -555,10 +555,10 @@ define @vwadd_vv_nxv2i64_nxv2i16( %va, @vwaddu_vv_nxv2i64_nxv2i16( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv2i64_nxv2i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v12, v9 -; CHECK-NEXT: vadd.vv v8, v10, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v8 +; CHECK-NEXT: vzext.vf2 v11, v9 +; CHECK-NEXT: vwaddu.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -571,10 +571,10 @@ define @vwadd_vx_nxv2i64_nxv2i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v8 -; CHECK-NEXT: vsext.vf4 v12, v9 -; CHECK-NEXT: vadd.vv v8, v10, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v8 +; CHECK-NEXT: vsext.vf2 v11, v9 +; CHECK-NEXT: vwadd.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -589,10 +589,10 @@ define @vwaddu_vx_nxv2i64_nxv2i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v12, v9 -; CHECK-NEXT: vadd.vv v8, v10, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v8 +; CHECK-NEXT: vzext.vf2 v11, v9 +; CHECK-NEXT: vwaddu.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -605,9 +605,9 @@ define @vwaddu_vx_nxv2i64_nxv2i16( %va, i16 define @vwadd_wv_nxv2i64_nxv2i16( %va, %vb) { ; CHECK-LABEL: vwadd_wv_nxv2i64_nxv2i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf4 v12, v10 -; CHECK-NEXT: vadd.vv v8, v8, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf2 v11, v10 +; CHECK-NEXT: vwadd.wv v8, v8, v11 ; CHECK-NEXT: ret %vc = sext %vb to %vd = add %va, %vc @@ -617,9 +617,9 @@ define @vwadd_wv_nxv2i64_nxv2i16( %va, @vwaddu_wv_nxv2i64_nxv2i16( %va, %vb) { ; CHECK-LABEL: vwaddu_wv_nxv2i64_nxv2i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vzext.vf4 v12, v10 -; CHECK-NEXT: vadd.vv v8, v8, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vzext.vf2 v11, v10 +; CHECK-NEXT: vwaddu.wv v8, v8, v11 ; CHECK-NEXT: ret %vc = zext %vb to %vd = add %va, %vc @@ -631,9 +631,9 @@ define @vwadd_wx_nxv2i64_nxv2i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf4 v12, v10 -; CHECK-NEXT: vadd.vv v8, v8, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf2 v11, v10 +; CHECK-NEXT: vwadd.wv v8, v8, v11 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -647,9 +647,9 @@ define @vwaddu_wx_nxv2i64_nxv2i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vzext.vf4 v12, v10 -; CHECK-NEXT: vadd.vv v8, v8, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vzext.vf2 v11, v10 +; CHECK-NEXT: vwaddu.wv v8, v8, v11 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -661,10 +661,10 @@ define @vwaddu_wx_nxv2i64_nxv2i16( %va, i16 define @vwadd_vv_nxv4i64_nxv4i16( %va, %vb) { ; CHECK-LABEL: vwadd_vv_nxv4i64_nxv4i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf4 v12, v8 -; CHECK-NEXT: vsext.vf4 v16, v9 -; CHECK-NEXT: vadd.vv v8, v12, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf2 v12, v8 +; CHECK-NEXT: vsext.vf2 v14, v9 +; CHECK-NEXT: vwadd.vv v8, v12, v14 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -675,10 +675,10 @@ define @vwadd_vv_nxv4i64_nxv4i16( %va, @vwaddu_vv_nxv4i64_nxv4i16( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv4i64_nxv4i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vzext.vf4 v12, v8 -; CHECK-NEXT: vzext.vf4 v16, v9 -; CHECK-NEXT: vadd.vv v8, v12, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v12, v8 +; CHECK-NEXT: vzext.vf2 v14, v9 +; CHECK-NEXT: vwaddu.vv v8, v12, v14 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -691,10 +691,10 @@ define @vwadd_vx_nxv4i64_nxv4i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf4 v12, v8 -; CHECK-NEXT: vsext.vf4 v16, v9 -; CHECK-NEXT: vadd.vv v8, v12, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf2 v12, v8 +; CHECK-NEXT: vsext.vf2 v14, v9 +; CHECK-NEXT: vwadd.vv v8, v12, v14 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -709,10 +709,10 @@ define @vwaddu_vx_nxv4i64_nxv4i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vzext.vf4 v12, v8 -; CHECK-NEXT: vzext.vf4 v16, v9 -; CHECK-NEXT: vadd.vv v8, v12, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v12, v8 +; CHECK-NEXT: vzext.vf2 v14, v9 +; CHECK-NEXT: vwaddu.vv v8, v12, v14 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -725,9 +725,9 @@ define @vwaddu_vx_nxv4i64_nxv4i16( %va, i16 define @vwadd_wv_nxv4i64_nxv4i16( %va, %vb) { ; CHECK-LABEL: vwadd_wv_nxv4i64_nxv4i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf4 v16, v12 -; CHECK-NEXT: vadd.vv v8, v8, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf2 v14, v12 +; CHECK-NEXT: vwadd.wv v8, v8, v14 ; CHECK-NEXT: ret %vc = sext %vb to %vd = add %va, %vc @@ -737,9 +737,9 @@ define @vwadd_wv_nxv4i64_nxv4i16( %va, @vwaddu_wv_nxv4i64_nxv4i16( %va, %vb) { ; CHECK-LABEL: vwaddu_wv_nxv4i64_nxv4i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vzext.vf4 v16, v12 -; CHECK-NEXT: vadd.vv v8, v8, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v14, v12 +; CHECK-NEXT: vwaddu.wv v8, v8, v14 ; CHECK-NEXT: ret %vc = zext %vb to %vd = add %va, %vc @@ -751,9 +751,9 @@ define @vwadd_wx_nxv4i64_nxv4i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma ; CHECK-NEXT: vmv.v.x v12, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf4 v16, v12 -; CHECK-NEXT: vadd.vv v8, v8, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf2 v14, v12 +; CHECK-NEXT: vwadd.wv v8, v8, v14 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -767,9 +767,9 @@ define @vwaddu_wx_nxv4i64_nxv4i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma ; CHECK-NEXT: vmv.v.x v12, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vzext.vf4 v16, v12 -; CHECK-NEXT: vadd.vv v8, v8, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v14, v12 +; CHECK-NEXT: vwaddu.wv v8, v8, v14 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -781,10 +781,10 @@ define @vwaddu_wx_nxv4i64_nxv4i16( %va, i16 define @vwadd_vv_nxv8i64_nxv8i16( %va, %vb) { ; CHECK-LABEL: vwadd_vv_nxv8i64_nxv8i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf4 v16, v8 -; CHECK-NEXT: vsext.vf4 v24, v10 -; CHECK-NEXT: vadd.vv v8, v16, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf2 v16, v8 +; CHECK-NEXT: vsext.vf2 v20, v10 +; CHECK-NEXT: vwadd.vv v8, v16, v20 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -795,10 +795,10 @@ define @vwadd_vv_nxv8i64_nxv8i16( %va, @vwaddu_vv_nxv8i64_nxv8i16( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv8i64_nxv8i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vzext.vf4 v16, v8 -; CHECK-NEXT: vzext.vf4 v24, v10 -; CHECK-NEXT: vadd.vv v8, v16, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vzext.vf2 v16, v8 +; CHECK-NEXT: vzext.vf2 v20, v10 +; CHECK-NEXT: vwaddu.vv v8, v16, v20 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -811,10 +811,10 @@ define @vwadd_vx_nxv8i64_nxv8i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma ; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf4 v16, v8 -; CHECK-NEXT: vsext.vf4 v24, v10 -; CHECK-NEXT: vadd.vv v8, v16, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf2 v16, v8 +; CHECK-NEXT: vsext.vf2 v20, v10 +; CHECK-NEXT: vwadd.vv v8, v16, v20 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -829,10 +829,10 @@ define @vwaddu_vx_nxv8i64_nxv8i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma ; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vzext.vf4 v16, v8 -; CHECK-NEXT: vzext.vf4 v24, v10 -; CHECK-NEXT: vadd.vv v8, v16, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vzext.vf2 v16, v8 +; CHECK-NEXT: vzext.vf2 v20, v10 +; CHECK-NEXT: vwaddu.vv v8, v16, v20 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -845,9 +845,9 @@ define @vwaddu_vx_nxv8i64_nxv8i16( %va, i16 define @vwadd_wv_nxv8i64_nxv8i16( %va, %vb) { ; CHECK-LABEL: vwadd_wv_nxv8i64_nxv8i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf4 v24, v16 -; CHECK-NEXT: vadd.vv v8, v8, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf2 v20, v16 +; CHECK-NEXT: vwadd.wv v8, v8, v20 ; CHECK-NEXT: ret %vc = sext %vb to %vd = add %va, %vc @@ -857,9 +857,9 @@ define @vwadd_wv_nxv8i64_nxv8i16( %va, @vwaddu_wv_nxv8i64_nxv8i16( %va, %vb) { ; CHECK-LABEL: vwaddu_wv_nxv8i64_nxv8i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vzext.vf4 v24, v16 -; CHECK-NEXT: vadd.vv v8, v8, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vzext.vf2 v20, v16 +; CHECK-NEXT: vwaddu.wv v8, v8, v20 ; CHECK-NEXT: ret %vc = zext %vb to %vd = add %va, %vc @@ -871,9 +871,9 @@ define @vwadd_wx_nxv8i64_nxv8i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma ; CHECK-NEXT: vmv.v.x v16, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf4 v24, v16 -; CHECK-NEXT: vadd.vv v8, v8, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf2 v20, v16 +; CHECK-NEXT: vwadd.wv v8, v8, v20 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -887,9 +887,9 @@ define @vwaddu_wx_nxv8i64_nxv8i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma ; CHECK-NEXT: vmv.v.x v16, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vzext.vf4 v24, v16 -; CHECK-NEXT: vadd.vv v8, v8, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vzext.vf2 v20, v16 +; CHECK-NEXT: vwaddu.wv v8, v8, v20 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -901,10 +901,10 @@ define @vwaddu_wx_nxv8i64_nxv8i16( %va, i16 define @vwadd_vv_nxv1i64_nxv1i8( %va, %vb) { ; CHECK-LABEL: vwadd_vv_nxv1i64_nxv1i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v8 -; CHECK-NEXT: vsext.vf8 v8, v9 -; CHECK-NEXT: vadd.vv v8, v10, v8 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vsext.vf4 v11, v9 +; CHECK-NEXT: vwadd.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -915,10 +915,10 @@ define @vwadd_vv_nxv1i64_nxv1i8( %va, @vwaddu_vv_nxv1i64_nxv1i8( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv1i64_nxv1i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vzext.vf8 v10, v8 -; CHECK-NEXT: vzext.vf8 v8, v9 -; CHECK-NEXT: vadd.vv v8, v10, v8 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf4 v10, v8 +; CHECK-NEXT: vzext.vf4 v11, v9 +; CHECK-NEXT: vwaddu.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -931,10 +931,10 @@ define @vwadd_vx_nxv1i64_nxv1i8( %va, i8 %b) ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v8 -; CHECK-NEXT: vsext.vf8 v8, v9 -; CHECK-NEXT: vadd.vv v8, v10, v8 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vsext.vf4 v11, v9 +; CHECK-NEXT: vwadd.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -949,10 +949,10 @@ define @vwaddu_vx_nxv1i64_nxv1i8( %va, i8 %b ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vzext.vf8 v10, v8 -; CHECK-NEXT: vzext.vf8 v8, v9 -; CHECK-NEXT: vadd.vv v8, v10, v8 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf4 v10, v8 +; CHECK-NEXT: vzext.vf4 v11, v9 +; CHECK-NEXT: vwaddu.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -965,9 +965,9 @@ define @vwaddu_vx_nxv1i64_nxv1i8( %va, i8 %b define @vwadd_wv_nxv1i64_nxv1i8( %va, %vb) { ; CHECK-LABEL: vwadd_wv_nxv1i64_nxv1i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v9 -; CHECK-NEXT: vadd.vv v8, v8, v10 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v9 +; CHECK-NEXT: vwadd.wv v8, v8, v10 ; CHECK-NEXT: ret %vc = sext %vb to %vd = add %va, %vc @@ -977,9 +977,9 @@ define @vwadd_wv_nxv1i64_nxv1i8( %va, @vwaddu_wv_nxv1i64_nxv1i8( %va, %vb) { ; CHECK-LABEL: vwaddu_wv_nxv1i64_nxv1i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vzext.vf8 v10, v9 -; CHECK-NEXT: vadd.vv v8, v8, v10 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf4 v10, v9 +; CHECK-NEXT: vwaddu.wv v8, v8, v10 ; CHECK-NEXT: ret %vc = zext %vb to %vd = add %va, %vc @@ -991,9 +991,9 @@ define @vwadd_wx_nxv1i64_nxv1i8( %va, i8 %b ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v9 -; CHECK-NEXT: vadd.vv v8, v8, v10 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v9 +; CHECK-NEXT: vwadd.wv v8, v8, v10 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1007,9 +1007,9 @@ define @vwaddu_wx_nxv1i64_nxv1i8( %va, i8 % ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vzext.vf8 v10, v9 -; CHECK-NEXT: vadd.vv v8, v8, v10 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf4 v10, v9 +; CHECK-NEXT: vwaddu.wv v8, v8, v10 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1021,10 +1021,10 @@ define @vwaddu_wx_nxv1i64_nxv1i8( %va, i8 % define @vwadd_vv_nxv2i64_nxv2i8( %va, %vb) { ; CHECK-LABEL: vwadd_vv_nxv2i64_nxv2i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v8 -; CHECK-NEXT: vsext.vf8 v12, v9 -; CHECK-NEXT: vadd.vv v8, v10, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vsext.vf4 v11, v9 +; CHECK-NEXT: vwadd.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -1035,10 +1035,10 @@ define @vwadd_vv_nxv2i64_nxv2i8( %va, @vwaddu_vv_nxv2i64_nxv2i8( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv2i64_nxv2i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vzext.vf8 v10, v8 -; CHECK-NEXT: vzext.vf8 v12, v9 -; CHECK-NEXT: vadd.vv v8, v10, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vzext.vf4 v10, v8 +; CHECK-NEXT: vzext.vf4 v11, v9 +; CHECK-NEXT: vwaddu.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -1051,10 +1051,10 @@ define @vwadd_vx_nxv2i64_nxv2i8( %va, i8 %b) ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v8 -; CHECK-NEXT: vsext.vf8 v12, v9 -; CHECK-NEXT: vadd.vv v8, v10, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vsext.vf4 v11, v9 +; CHECK-NEXT: vwadd.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1069,10 +1069,10 @@ define @vwaddu_vx_nxv2i64_nxv2i8( %va, i8 %b ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vzext.vf8 v10, v8 -; CHECK-NEXT: vzext.vf8 v12, v9 -; CHECK-NEXT: vadd.vv v8, v10, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vzext.vf4 v10, v8 +; CHECK-NEXT: vzext.vf4 v11, v9 +; CHECK-NEXT: vwaddu.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1085,9 +1085,9 @@ define @vwaddu_vx_nxv2i64_nxv2i8( %va, i8 %b define @vwadd_wv_nxv2i64_nxv2i8( %va, %vb) { ; CHECK-LABEL: vwadd_wv_nxv2i64_nxv2i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf8 v12, v10 -; CHECK-NEXT: vadd.vv v8, v8, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf4 v11, v10 +; CHECK-NEXT: vwadd.wv v8, v8, v11 ; CHECK-NEXT: ret %vc = sext %vb to %vd = add %va, %vc @@ -1097,9 +1097,9 @@ define @vwadd_wv_nxv2i64_nxv2i8( %va, @vwaddu_wv_nxv2i64_nxv2i8( %va, %vb) { ; CHECK-LABEL: vwaddu_wv_nxv2i64_nxv2i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vzext.vf8 v12, v10 -; CHECK-NEXT: vadd.vv v8, v8, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vzext.vf4 v11, v10 +; CHECK-NEXT: vwaddu.wv v8, v8, v11 ; CHECK-NEXT: ret %vc = zext %vb to %vd = add %va, %vc @@ -1111,9 +1111,9 @@ define @vwadd_wx_nxv2i64_nxv2i8( %va, i8 %b ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf8 v12, v10 -; CHECK-NEXT: vadd.vv v8, v8, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf4 v11, v10 +; CHECK-NEXT: vwadd.wv v8, v8, v11 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1127,9 +1127,9 @@ define @vwaddu_wx_nxv2i64_nxv2i8( %va, i8 % ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vzext.vf8 v12, v10 -; CHECK-NEXT: vadd.vv v8, v8, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vzext.vf4 v11, v10 +; CHECK-NEXT: vwaddu.wv v8, v8, v11 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1141,10 +1141,10 @@ define @vwaddu_wx_nxv2i64_nxv2i8( %va, i8 % define @vwadd_vv_nxv4i64_nxv4i8( %va, %vb) { ; CHECK-LABEL: vwadd_vv_nxv4i64_nxv4i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf8 v12, v8 -; CHECK-NEXT: vsext.vf8 v16, v9 -; CHECK-NEXT: vadd.vv v8, v12, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf4 v12, v8 +; CHECK-NEXT: vsext.vf4 v14, v9 +; CHECK-NEXT: vwadd.vv v8, v12, v14 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -1155,10 +1155,10 @@ define @vwadd_vv_nxv4i64_nxv4i8( %va, @vwaddu_vv_nxv4i64_nxv4i8( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv4i64_nxv4i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vzext.vf8 v12, v8 -; CHECK-NEXT: vzext.vf8 v16, v9 -; CHECK-NEXT: vadd.vv v8, v12, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf4 v12, v8 +; CHECK-NEXT: vzext.vf4 v14, v9 +; CHECK-NEXT: vwaddu.vv v8, v12, v14 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -1171,10 +1171,10 @@ define @vwadd_vx_nxv4i64_nxv4i8( %va, i8 %b) ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf8 v12, v8 -; CHECK-NEXT: vsext.vf8 v16, v9 -; CHECK-NEXT: vadd.vv v8, v12, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf4 v12, v8 +; CHECK-NEXT: vsext.vf4 v14, v9 +; CHECK-NEXT: vwadd.vv v8, v12, v14 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1189,10 +1189,10 @@ define @vwaddu_vx_nxv4i64_nxv4i8( %va, i8 %b ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vzext.vf8 v12, v8 -; CHECK-NEXT: vzext.vf8 v16, v9 -; CHECK-NEXT: vadd.vv v8, v12, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf4 v12, v8 +; CHECK-NEXT: vzext.vf4 v14, v9 +; CHECK-NEXT: vwaddu.vv v8, v12, v14 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1205,9 +1205,9 @@ define @vwaddu_vx_nxv4i64_nxv4i8( %va, i8 %b define @vwadd_wv_nxv4i64_nxv4i8( %va, %vb) { ; CHECK-LABEL: vwadd_wv_nxv4i64_nxv4i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf8 v16, v12 -; CHECK-NEXT: vadd.vv v8, v8, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf4 v14, v12 +; CHECK-NEXT: vwadd.wv v8, v8, v14 ; CHECK-NEXT: ret %vc = sext %vb to %vd = add %va, %vc @@ -1217,9 +1217,9 @@ define @vwadd_wv_nxv4i64_nxv4i8( %va, @vwaddu_wv_nxv4i64_nxv4i8( %va, %vb) { ; CHECK-LABEL: vwaddu_wv_nxv4i64_nxv4i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vzext.vf8 v16, v12 -; CHECK-NEXT: vadd.vv v8, v8, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf4 v14, v12 +; CHECK-NEXT: vwaddu.wv v8, v8, v14 ; CHECK-NEXT: ret %vc = zext %vb to %vd = add %va, %vc @@ -1231,9 +1231,9 @@ define @vwadd_wx_nxv4i64_nxv4i8( %va, i8 %b ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v12, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf8 v16, v12 -; CHECK-NEXT: vadd.vv v8, v8, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf4 v14, v12 +; CHECK-NEXT: vwadd.wv v8, v8, v14 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1247,9 +1247,9 @@ define @vwaddu_wx_nxv4i64_nxv4i8( %va, i8 % ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v12, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vzext.vf8 v16, v12 -; CHECK-NEXT: vadd.vv v8, v8, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf4 v14, v12 +; CHECK-NEXT: vwaddu.wv v8, v8, v14 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1261,10 +1261,10 @@ define @vwaddu_wx_nxv4i64_nxv4i8( %va, i8 % define @vwadd_vv_nxv8i64_nxv8i8( %va, %vb) { ; CHECK-LABEL: vwadd_vv_nxv8i64_nxv8i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vsext.vf8 v24, v9 -; CHECK-NEXT: vadd.vv v8, v16, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf4 v16, v8 +; CHECK-NEXT: vsext.vf4 v20, v9 +; CHECK-NEXT: vwadd.vv v8, v16, v20 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -1275,10 +1275,10 @@ define @vwadd_vv_nxv8i64_nxv8i8( %va, @vwaddu_vv_nxv8i64_nxv8i8( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv8i64_nxv8i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vzext.vf8 v16, v8 -; CHECK-NEXT: vzext.vf8 v24, v9 -; CHECK-NEXT: vadd.vv v8, v16, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vzext.vf4 v16, v8 +; CHECK-NEXT: vzext.vf4 v20, v9 +; CHECK-NEXT: vwaddu.vv v8, v16, v20 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -1291,10 +1291,10 @@ define @vwadd_vx_nxv8i64_nxv8i8( %va, i8 %b) ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vsext.vf8 v24, v9 -; CHECK-NEXT: vadd.vv v8, v16, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf4 v16, v8 +; CHECK-NEXT: vsext.vf4 v20, v9 +; CHECK-NEXT: vwadd.vv v8, v16, v20 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1309,10 +1309,10 @@ define @vwaddu_vx_nxv8i64_nxv8i8( %va, i8 %b ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vzext.vf8 v16, v8 -; CHECK-NEXT: vzext.vf8 v24, v9 -; CHECK-NEXT: vadd.vv v8, v16, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vzext.vf4 v16, v8 +; CHECK-NEXT: vzext.vf4 v20, v9 +; CHECK-NEXT: vwaddu.vv v8, v16, v20 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1325,9 +1325,9 @@ define @vwaddu_vx_nxv8i64_nxv8i8( %va, i8 %b define @vwadd_wv_nxv8i64_nxv8i8( %va, %vb) { ; CHECK-LABEL: vwadd_wv_nxv8i64_nxv8i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf8 v24, v16 -; CHECK-NEXT: vadd.vv v8, v8, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf4 v20, v16 +; CHECK-NEXT: vwadd.wv v8, v8, v20 ; CHECK-NEXT: ret %vc = sext %vb to %vd = add %va, %vc @@ -1337,9 +1337,9 @@ define @vwadd_wv_nxv8i64_nxv8i8( %va, @vwaddu_wv_nxv8i64_nxv8i8( %va, %vb) { ; CHECK-LABEL: vwaddu_wv_nxv8i64_nxv8i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vzext.vf8 v24, v16 -; CHECK-NEXT: vadd.vv v8, v8, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vzext.vf4 v20, v16 +; CHECK-NEXT: vwaddu.wv v8, v8, v20 ; CHECK-NEXT: ret %vc = zext %vb to %vd = add %va, %vc @@ -1351,9 +1351,9 @@ define @vwadd_wx_nxv8i64_nxv8i8( %va, i8 %b ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma ; CHECK-NEXT: vmv.v.x v16, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf8 v24, v16 -; CHECK-NEXT: vadd.vv v8, v8, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf4 v20, v16 +; CHECK-NEXT: vwadd.wv v8, v8, v20 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1367,9 +1367,9 @@ define @vwaddu_wx_nxv8i64_nxv8i8( %va, i8 % ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma ; CHECK-NEXT: vmv.v.x v16, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vzext.vf8 v24, v16 -; CHECK-NEXT: vadd.vv v8, v8, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vzext.vf4 v20, v16 +; CHECK-NEXT: vwaddu.wv v8, v8, v20 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1377,3 +1377,34 @@ define @vwaddu_wx_nxv8i64_nxv8i8( %va, i8 % %vc = add %va, %vb ret %vc } + +; Make sure that we don't introduce any V{S,Z}EXT_VL nodes with i1 types from +; combineBinOp_VLToVWBinOp_VL, since they can't be selected. +define @i1_zext( %va, %vb, ptr %p) { +; RV32-LABEL: i1_zext: +; RV32: # %bb.0: +; RV32-NEXT: vsetvli a1, zero, e64, m1, ta, ma +; RV32-NEXT: vmv.v.i v9, 0 +; RV32-NEXT: vmerge.vim v9, v9, 1, v0 +; RV32-NEXT: vadd.vv v8, v9, v8 +; RV32-NEXT: li a1, 42 +; RV32-NEXT: sh a1, 0(a0) +; RV32-NEXT: ret +; +; RV64-LABEL: i1_zext: +; RV64: # %bb.0: +; RV64-NEXT: vsetvli a1, zero, e64, m1, ta, mu +; RV64-NEXT: vadd.vi v8, v8, 1, v0.t +; RV64-NEXT: li a1, 42 +; RV64-NEXT: sh a1, 0(a0) +; RV64-NEXT: ret + %vc = zext %va to + %vd = add %vc, %vb + +; Introduce an illegal type so that the DAG changes after legalizing +; types. Otherwise the legalize vector ops phase will be run immediately after +; the legalize types phase, and the zext will already be in non-i1 form by the +; time combineBinOp_VLToVWBinOp_VL is called. + store i9 42, ptr %p + ret %vd +} diff --git a/llvm/test/CodeGen/RISCV/rvv/vwmul-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vwmul-sdnode.ll index 3634162eefd6..539a4bdb27ad 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vwmul-sdnode.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vwmul-sdnode.ll @@ -341,10 +341,10 @@ define @vwmulsu_vx_nxv8i64_nxv8i32( %va, i3 define @vwmul_vv_nxv1i64_nxv1i16( %va, %vb) { ; CHECK-LABEL: vwmul_vv_nxv1i64_nxv1i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v8 -; CHECK-NEXT: vsext.vf4 v8, v9 -; CHECK-NEXT: vmul.vv v8, v10, v8 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v8 +; CHECK-NEXT: vsext.vf2 v11, v9 +; CHECK-NEXT: vwmul.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -355,10 +355,10 @@ define @vwmul_vv_nxv1i64_nxv1i16( %va, @vwmulu_vv_nxv1i64_nxv1i16( %va, %vb) { ; CHECK-LABEL: vwmulu_vv_nxv1i64_nxv1i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v8, v9 -; CHECK-NEXT: vmul.vv v8, v10, v8 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v8 +; CHECK-NEXT: vzext.vf2 v11, v9 +; CHECK-NEXT: vwmulu.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -369,10 +369,10 @@ define @vwmulu_vv_nxv1i64_nxv1i16( %va, @vwmulsu_vv_nxv1i64_nxv1i16( %va, %vb) { ; CHECK-LABEL: vwmulsu_vv_nxv1i64_nxv1i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v8, v9 -; CHECK-NEXT: vmul.vv v8, v10, v8 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v8 +; CHECK-NEXT: vzext.vf2 v11, v9 +; CHECK-NEXT: vwmulsu.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = sext %va to %vd = zext %vb to @@ -385,10 +385,10 @@ define @vwmul_vx_nxv1i64_nxv1i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v8 -; CHECK-NEXT: vsext.vf4 v8, v9 -; CHECK-NEXT: vmul.vv v8, v10, v8 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v8 +; CHECK-NEXT: vsext.vf2 v11, v9 +; CHECK-NEXT: vwmul.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement undef, i16 %b, i16 0 %splat = shufflevector %head, undef, zeroinitializer @@ -403,10 +403,10 @@ define @vwmulu_vx_nxv1i64_nxv1i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v8, v9 -; CHECK-NEXT: vmul.vv v8, v10, v8 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v8 +; CHECK-NEXT: vzext.vf2 v11, v9 +; CHECK-NEXT: vwmulu.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement undef, i16 %b, i16 0 %splat = shufflevector %head, undef, zeroinitializer @@ -421,10 +421,10 @@ define @vwmulsu_vx_nxv1i64_nxv1i16( %va, i1 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v8, v9 -; CHECK-NEXT: vmul.vv v8, v10, v8 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v8 +; CHECK-NEXT: vzext.vf2 v11, v9 +; CHECK-NEXT: vwmulsu.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement undef, i16 %b, i16 0 %splat = shufflevector %head, undef, zeroinitializer @@ -437,10 +437,10 @@ define @vwmulsu_vx_nxv1i64_nxv1i16( %va, i1 define @vwmul_vv_nxv2i64_nxv2i16( %va, %vb) { ; CHECK-LABEL: vwmul_vv_nxv2i64_nxv2i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v8 -; CHECK-NEXT: vsext.vf4 v12, v9 -; CHECK-NEXT: vmul.vv v8, v10, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v8 +; CHECK-NEXT: vsext.vf2 v11, v9 +; CHECK-NEXT: vwmul.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -451,10 +451,10 @@ define @vwmul_vv_nxv2i64_nxv2i16( %va, @vwmulu_vv_nxv2i64_nxv2i16( %va, %vb) { ; CHECK-LABEL: vwmulu_vv_nxv2i64_nxv2i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v12, v9 -; CHECK-NEXT: vmul.vv v8, v10, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v8 +; CHECK-NEXT: vzext.vf2 v11, v9 +; CHECK-NEXT: vwmulu.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -465,10 +465,10 @@ define @vwmulu_vv_nxv2i64_nxv2i16( %va, @vwmulsu_vv_nxv2i64_nxv2i16( %va, %vb) { ; CHECK-LABEL: vwmulsu_vv_nxv2i64_nxv2i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v12, v9 -; CHECK-NEXT: vmul.vv v8, v10, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v8 +; CHECK-NEXT: vzext.vf2 v11, v9 +; CHECK-NEXT: vwmulsu.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = sext %va to %vd = zext %vb to @@ -481,10 +481,10 @@ define @vwmul_vx_nxv2i64_nxv2i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v8 -; CHECK-NEXT: vsext.vf4 v12, v9 -; CHECK-NEXT: vmul.vv v8, v10, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v8 +; CHECK-NEXT: vsext.vf2 v11, v9 +; CHECK-NEXT: vwmul.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement undef, i16 %b, i16 0 %splat = shufflevector %head, undef, zeroinitializer @@ -499,10 +499,10 @@ define @vwmulu_vx_nxv2i64_nxv2i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v12, v9 -; CHECK-NEXT: vmul.vv v8, v10, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v8 +; CHECK-NEXT: vzext.vf2 v11, v9 +; CHECK-NEXT: vwmulu.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement undef, i16 %b, i16 0 %splat = shufflevector %head, undef, zeroinitializer @@ -517,10 +517,10 @@ define @vwmulsu_vx_nxv2i64_nxv2i16( %va, i1 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v12, v9 -; CHECK-NEXT: vmul.vv v8, v10, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v8 +; CHECK-NEXT: vzext.vf2 v11, v9 +; CHECK-NEXT: vwmulsu.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement undef, i16 %b, i16 0 %splat = shufflevector %head, undef, zeroinitializer @@ -533,10 +533,10 @@ define @vwmulsu_vx_nxv2i64_nxv2i16( %va, i1 define @vwmul_vv_nxv4i64_nxv4i16( %va, %vb) { ; CHECK-LABEL: vwmul_vv_nxv4i64_nxv4i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf4 v12, v8 -; CHECK-NEXT: vsext.vf4 v16, v9 -; CHECK-NEXT: vmul.vv v8, v12, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf2 v12, v8 +; CHECK-NEXT: vsext.vf2 v14, v9 +; CHECK-NEXT: vwmul.vv v8, v12, v14 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -547,10 +547,10 @@ define @vwmul_vv_nxv4i64_nxv4i16( %va, @vwmulu_vv_nxv4i64_nxv4i16( %va, %vb) { ; CHECK-LABEL: vwmulu_vv_nxv4i64_nxv4i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vzext.vf4 v12, v8 -; CHECK-NEXT: vzext.vf4 v16, v9 -; CHECK-NEXT: vmul.vv v8, v12, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v12, v8 +; CHECK-NEXT: vzext.vf2 v14, v9 +; CHECK-NEXT: vwmulu.vv v8, v12, v14 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -561,10 +561,10 @@ define @vwmulu_vv_nxv4i64_nxv4i16( %va, @vwmulsu_vv_nxv4i64_nxv4i16( %va, %vb) { ; CHECK-LABEL: vwmulsu_vv_nxv4i64_nxv4i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf4 v12, v8 -; CHECK-NEXT: vzext.vf4 v16, v9 -; CHECK-NEXT: vmul.vv v8, v12, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf2 v12, v8 +; CHECK-NEXT: vzext.vf2 v14, v9 +; CHECK-NEXT: vwmulsu.vv v8, v12, v14 ; CHECK-NEXT: ret %vc = sext %va to %vd = zext %vb to @@ -577,10 +577,10 @@ define @vwmul_vx_nxv4i64_nxv4i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf4 v12, v8 -; CHECK-NEXT: vsext.vf4 v16, v9 -; CHECK-NEXT: vmul.vv v8, v12, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf2 v12, v8 +; CHECK-NEXT: vsext.vf2 v14, v9 +; CHECK-NEXT: vwmul.vv v8, v12, v14 ; CHECK-NEXT: ret %head = insertelement undef, i16 %b, i16 0 %splat = shufflevector %head, undef, zeroinitializer @@ -595,10 +595,10 @@ define @vwmulu_vx_nxv4i64_nxv4i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vzext.vf4 v12, v8 -; CHECK-NEXT: vzext.vf4 v16, v9 -; CHECK-NEXT: vmul.vv v8, v12, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v12, v8 +; CHECK-NEXT: vzext.vf2 v14, v9 +; CHECK-NEXT: vwmulu.vv v8, v12, v14 ; CHECK-NEXT: ret %head = insertelement undef, i16 %b, i16 0 %splat = shufflevector %head, undef, zeroinitializer @@ -613,10 +613,10 @@ define @vwmulsu_vx_nxv4i64_nxv4i16( %va, i1 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf4 v12, v8 -; CHECK-NEXT: vzext.vf4 v16, v9 -; CHECK-NEXT: vmul.vv v8, v12, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf2 v12, v8 +; CHECK-NEXT: vzext.vf2 v14, v9 +; CHECK-NEXT: vwmulsu.vv v8, v12, v14 ; CHECK-NEXT: ret %head = insertelement undef, i16 %b, i16 0 %splat = shufflevector %head, undef, zeroinitializer @@ -629,10 +629,10 @@ define @vwmulsu_vx_nxv4i64_nxv4i16( %va, i1 define @vwmul_vv_nxv8i64_nxv8i16( %va, %vb) { ; CHECK-LABEL: vwmul_vv_nxv8i64_nxv8i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf4 v16, v8 -; CHECK-NEXT: vsext.vf4 v24, v10 -; CHECK-NEXT: vmul.vv v8, v16, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf2 v16, v8 +; CHECK-NEXT: vsext.vf2 v20, v10 +; CHECK-NEXT: vwmul.vv v8, v16, v20 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -643,10 +643,10 @@ define @vwmul_vv_nxv8i64_nxv8i16( %va, @vwmulu_vv_nxv8i64_nxv8i16( %va, %vb) { ; CHECK-LABEL: vwmulu_vv_nxv8i64_nxv8i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vzext.vf4 v16, v8 -; CHECK-NEXT: vzext.vf4 v24, v10 -; CHECK-NEXT: vmul.vv v8, v16, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vzext.vf2 v16, v8 +; CHECK-NEXT: vzext.vf2 v20, v10 +; CHECK-NEXT: vwmulu.vv v8, v16, v20 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -657,10 +657,10 @@ define @vwmulu_vv_nxv8i64_nxv8i16( %va, @vwmulsu_vv_nxv8i64_nxv8i16( %va, %vb) { ; CHECK-LABEL: vwmulsu_vv_nxv8i64_nxv8i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf4 v16, v8 -; CHECK-NEXT: vzext.vf4 v24, v10 -; CHECK-NEXT: vmul.vv v8, v16, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf2 v16, v8 +; CHECK-NEXT: vzext.vf2 v20, v10 +; CHECK-NEXT: vwmulsu.vv v8, v16, v20 ; CHECK-NEXT: ret %vc = sext %va to %vd = zext %vb to @@ -673,10 +673,10 @@ define @vwmul_vx_nxv8i64_nxv8i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma ; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf4 v16, v8 -; CHECK-NEXT: vsext.vf4 v24, v10 -; CHECK-NEXT: vmul.vv v8, v16, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf2 v16, v8 +; CHECK-NEXT: vsext.vf2 v20, v10 +; CHECK-NEXT: vwmul.vv v8, v16, v20 ; CHECK-NEXT: ret %head = insertelement undef, i16 %b, i16 0 %splat = shufflevector %head, undef, zeroinitializer @@ -691,10 +691,10 @@ define @vwmulu_vx_nxv8i64_nxv8i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma ; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vzext.vf4 v16, v8 -; CHECK-NEXT: vzext.vf4 v24, v10 -; CHECK-NEXT: vmul.vv v8, v16, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vzext.vf2 v16, v8 +; CHECK-NEXT: vzext.vf2 v20, v10 +; CHECK-NEXT: vwmulu.vv v8, v16, v20 ; CHECK-NEXT: ret %head = insertelement undef, i16 %b, i16 0 %splat = shufflevector %head, undef, zeroinitializer @@ -709,10 +709,10 @@ define @vwmulsu_vx_nxv8i64_nxv8i16( %va, i1 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma ; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf4 v16, v8 -; CHECK-NEXT: vzext.vf4 v24, v10 -; CHECK-NEXT: vmul.vv v8, v16, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf2 v16, v8 +; CHECK-NEXT: vzext.vf2 v20, v10 +; CHECK-NEXT: vwmulsu.vv v8, v16, v20 ; CHECK-NEXT: ret %head = insertelement undef, i16 %b, i16 0 %splat = shufflevector %head, undef, zeroinitializer @@ -725,10 +725,10 @@ define @vwmulsu_vx_nxv8i64_nxv8i16( %va, i1 define @vwmul_vv_nxv1i64_nxv1i8( %va, %vb) { ; CHECK-LABEL: vwmul_vv_nxv1i64_nxv1i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v8 -; CHECK-NEXT: vsext.vf8 v8, v9 -; CHECK-NEXT: vmul.vv v8, v10, v8 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vsext.vf4 v11, v9 +; CHECK-NEXT: vwmul.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -739,10 +739,10 @@ define @vwmul_vv_nxv1i64_nxv1i8( %va, @vwmulu_vv_nxv1i64_nxv1i8( %va, %vb) { ; CHECK-LABEL: vwmulu_vv_nxv1i64_nxv1i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vzext.vf8 v10, v8 -; CHECK-NEXT: vzext.vf8 v8, v9 -; CHECK-NEXT: vmul.vv v8, v10, v8 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf4 v10, v8 +; CHECK-NEXT: vzext.vf4 v11, v9 +; CHECK-NEXT: vwmulu.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -753,10 +753,10 @@ define @vwmulu_vv_nxv1i64_nxv1i8( %va, @vwmulsu_vv_nxv1i64_nxv1i8( %va, %vb) { ; CHECK-LABEL: vwmulsu_vv_nxv1i64_nxv1i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v8 -; CHECK-NEXT: vzext.vf8 v8, v9 -; CHECK-NEXT: vmul.vv v8, v10, v8 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vzext.vf4 v11, v9 +; CHECK-NEXT: vwmulsu.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = sext %va to %vd = zext %vb to @@ -769,10 +769,10 @@ define @vwmul_vx_nxv1i64_nxv1i8( %va, i8 %b) ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v8 -; CHECK-NEXT: vsext.vf8 v8, v9 -; CHECK-NEXT: vmul.vv v8, v10, v8 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vsext.vf4 v11, v9 +; CHECK-NEXT: vwmul.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement undef, i8 %b, i8 0 %splat = shufflevector %head, undef, zeroinitializer @@ -787,10 +787,10 @@ define @vwmulu_vx_nxv1i64_nxv1i8( %va, i8 %b ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vzext.vf8 v10, v8 -; CHECK-NEXT: vzext.vf8 v8, v9 -; CHECK-NEXT: vmul.vv v8, v10, v8 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf4 v10, v8 +; CHECK-NEXT: vzext.vf4 v11, v9 +; CHECK-NEXT: vwmulu.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement undef, i8 %b, i8 0 %splat = shufflevector %head, undef, zeroinitializer @@ -805,10 +805,10 @@ define @vwmulsu_vx_nxv1i64_nxv1i8( %va, i8 % ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v8 -; CHECK-NEXT: vzext.vf8 v8, v9 -; CHECK-NEXT: vmul.vv v8, v10, v8 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vzext.vf4 v11, v9 +; CHECK-NEXT: vwmulsu.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement undef, i8 %b, i8 0 %splat = shufflevector %head, undef, zeroinitializer @@ -821,10 +821,10 @@ define @vwmulsu_vx_nxv1i64_nxv1i8( %va, i8 % define @vwmul_vv_nxv2i64_nxv2i8( %va, %vb) { ; CHECK-LABEL: vwmul_vv_nxv2i64_nxv2i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v8 -; CHECK-NEXT: vsext.vf8 v12, v9 -; CHECK-NEXT: vmul.vv v8, v10, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vsext.vf4 v11, v9 +; CHECK-NEXT: vwmul.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -835,10 +835,10 @@ define @vwmul_vv_nxv2i64_nxv2i8( %va, @vwmulu_vv_nxv2i64_nxv2i8( %va, %vb) { ; CHECK-LABEL: vwmulu_vv_nxv2i64_nxv2i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vzext.vf8 v10, v8 -; CHECK-NEXT: vzext.vf8 v12, v9 -; CHECK-NEXT: vmul.vv v8, v10, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vzext.vf4 v10, v8 +; CHECK-NEXT: vzext.vf4 v11, v9 +; CHECK-NEXT: vwmulu.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -849,10 +849,10 @@ define @vwmulu_vv_nxv2i64_nxv2i8( %va, @vwmulsu_vv_nxv2i64_nxv2i8( %va, %vb) { ; CHECK-LABEL: vwmulsu_vv_nxv2i64_nxv2i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v8 -; CHECK-NEXT: vzext.vf8 v12, v9 -; CHECK-NEXT: vmul.vv v8, v10, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vzext.vf4 v11, v9 +; CHECK-NEXT: vwmulsu.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = sext %va to %vd = zext %vb to @@ -865,10 +865,10 @@ define @vwmul_vx_nxv2i64_nxv2i8( %va, i8 %b) ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v8 -; CHECK-NEXT: vsext.vf8 v12, v9 -; CHECK-NEXT: vmul.vv v8, v10, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vsext.vf4 v11, v9 +; CHECK-NEXT: vwmul.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement undef, i8 %b, i8 0 %splat = shufflevector %head, undef, zeroinitializer @@ -883,10 +883,10 @@ define @vwmulu_vx_nxv2i64_nxv2i8( %va, i8 %b ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vzext.vf8 v10, v8 -; CHECK-NEXT: vzext.vf8 v12, v9 -; CHECK-NEXT: vmul.vv v8, v10, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vzext.vf4 v10, v8 +; CHECK-NEXT: vzext.vf4 v11, v9 +; CHECK-NEXT: vwmulu.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement undef, i8 %b, i8 0 %splat = shufflevector %head, undef, zeroinitializer @@ -901,10 +901,10 @@ define @vwmulsu_vx_nxv2i64_nxv2i8( %va, i8 % ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v8 -; CHECK-NEXT: vzext.vf8 v12, v9 -; CHECK-NEXT: vmul.vv v8, v10, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vzext.vf4 v11, v9 +; CHECK-NEXT: vwmulsu.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement undef, i8 %b, i8 0 %splat = shufflevector %head, undef, zeroinitializer @@ -917,10 +917,10 @@ define @vwmulsu_vx_nxv2i64_nxv2i8( %va, i8 % define @vwmul_vv_nxv4i64_nxv4i8( %va, %vb) { ; CHECK-LABEL: vwmul_vv_nxv4i64_nxv4i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf8 v12, v8 -; CHECK-NEXT: vsext.vf8 v16, v9 -; CHECK-NEXT: vmul.vv v8, v12, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf4 v12, v8 +; CHECK-NEXT: vsext.vf4 v14, v9 +; CHECK-NEXT: vwmul.vv v8, v12, v14 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -931,10 +931,10 @@ define @vwmul_vv_nxv4i64_nxv4i8( %va, @vwmulu_vv_nxv4i64_nxv4i8( %va, %vb) { ; CHECK-LABEL: vwmulu_vv_nxv4i64_nxv4i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vzext.vf8 v12, v8 -; CHECK-NEXT: vzext.vf8 v16, v9 -; CHECK-NEXT: vmul.vv v8, v12, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf4 v12, v8 +; CHECK-NEXT: vzext.vf4 v14, v9 +; CHECK-NEXT: vwmulu.vv v8, v12, v14 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -945,10 +945,10 @@ define @vwmulu_vv_nxv4i64_nxv4i8( %va, @vwmulsu_vv_nxv4i64_nxv4i8( %va, %vb) { ; CHECK-LABEL: vwmulsu_vv_nxv4i64_nxv4i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf8 v12, v8 -; CHECK-NEXT: vzext.vf8 v16, v9 -; CHECK-NEXT: vmul.vv v8, v12, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf4 v12, v8 +; CHECK-NEXT: vzext.vf4 v14, v9 +; CHECK-NEXT: vwmulsu.vv v8, v12, v14 ; CHECK-NEXT: ret %vc = sext %va to %vd = zext %vb to @@ -961,10 +961,10 @@ define @vwmul_vx_nxv4i64_nxv4i8( %va, i8 %b) ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf8 v12, v8 -; CHECK-NEXT: vsext.vf8 v16, v9 -; CHECK-NEXT: vmul.vv v8, v12, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf4 v12, v8 +; CHECK-NEXT: vsext.vf4 v14, v9 +; CHECK-NEXT: vwmul.vv v8, v12, v14 ; CHECK-NEXT: ret %head = insertelement undef, i8 %b, i8 0 %splat = shufflevector %head, undef, zeroinitializer @@ -979,10 +979,10 @@ define @vwmulu_vx_nxv4i64_nxv4i8( %va, i8 %b ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vzext.vf8 v12, v8 -; CHECK-NEXT: vzext.vf8 v16, v9 -; CHECK-NEXT: vmul.vv v8, v12, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf4 v12, v8 +; CHECK-NEXT: vzext.vf4 v14, v9 +; CHECK-NEXT: vwmulu.vv v8, v12, v14 ; CHECK-NEXT: ret %head = insertelement undef, i8 %b, i8 0 %splat = shufflevector %head, undef, zeroinitializer @@ -997,10 +997,10 @@ define @vwmulsu_vx_nxv4i64_nxv4i8( %va, i8 % ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf8 v12, v8 -; CHECK-NEXT: vzext.vf8 v16, v9 -; CHECK-NEXT: vmul.vv v8, v12, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf4 v12, v8 +; CHECK-NEXT: vzext.vf4 v14, v9 +; CHECK-NEXT: vwmulsu.vv v8, v12, v14 ; CHECK-NEXT: ret %head = insertelement undef, i8 %b, i8 0 %splat = shufflevector %head, undef, zeroinitializer @@ -1013,10 +1013,10 @@ define @vwmulsu_vx_nxv4i64_nxv4i8( %va, i8 % define @vwmul_vv_nxv8i64_nxv8i8( %va, %vb) { ; CHECK-LABEL: vwmul_vv_nxv8i64_nxv8i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vsext.vf8 v24, v9 -; CHECK-NEXT: vmul.vv v8, v16, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf4 v16, v8 +; CHECK-NEXT: vsext.vf4 v20, v9 +; CHECK-NEXT: vwmul.vv v8, v16, v20 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -1027,10 +1027,10 @@ define @vwmul_vv_nxv8i64_nxv8i8( %va, @vwmulu_vv_nxv8i64_nxv8i8( %va, %vb) { ; CHECK-LABEL: vwmulu_vv_nxv8i64_nxv8i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vzext.vf8 v16, v8 -; CHECK-NEXT: vzext.vf8 v24, v9 -; CHECK-NEXT: vmul.vv v8, v16, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vzext.vf4 v16, v8 +; CHECK-NEXT: vzext.vf4 v20, v9 +; CHECK-NEXT: vwmulu.vv v8, v16, v20 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -1041,10 +1041,10 @@ define @vwmulu_vv_nxv8i64_nxv8i8( %va, @vwmulsu_vv_nxv8i64_nxv8i8( %va, %vb) { ; CHECK-LABEL: vwmulsu_vv_nxv8i64_nxv8i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vzext.vf8 v24, v9 -; CHECK-NEXT: vmul.vv v8, v16, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf4 v16, v8 +; CHECK-NEXT: vzext.vf4 v20, v9 +; CHECK-NEXT: vwmulsu.vv v8, v16, v20 ; CHECK-NEXT: ret %vc = sext %va to %vd = zext %vb to @@ -1057,10 +1057,10 @@ define @vwmul_vx_nxv8i64_nxv8i8( %va, i8 %b) ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vsext.vf8 v24, v9 -; CHECK-NEXT: vmul.vv v8, v16, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf4 v16, v8 +; CHECK-NEXT: vsext.vf4 v20, v9 +; CHECK-NEXT: vwmul.vv v8, v16, v20 ; CHECK-NEXT: ret %head = insertelement undef, i8 %b, i8 0 %splat = shufflevector %head, undef, zeroinitializer @@ -1075,10 +1075,10 @@ define @vwmulu_vx_nxv8i64_nxv8i8( %va, i8 %b ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vzext.vf8 v16, v8 -; CHECK-NEXT: vzext.vf8 v24, v9 -; CHECK-NEXT: vmul.vv v8, v16, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vzext.vf4 v16, v8 +; CHECK-NEXT: vzext.vf4 v20, v9 +; CHECK-NEXT: vwmulu.vv v8, v16, v20 ; CHECK-NEXT: ret %head = insertelement undef, i8 %b, i8 0 %splat = shufflevector %head, undef, zeroinitializer @@ -1093,10 +1093,10 @@ define @vwmulsu_vx_nxv8i64_nxv8i8( %va, i8 % ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vzext.vf8 v24, v9 -; CHECK-NEXT: vmul.vv v8, v16, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf4 v16, v8 +; CHECK-NEXT: vzext.vf4 v20, v9 +; CHECK-NEXT: vwmulsu.vv v8, v16, v20 ; CHECK-NEXT: ret %head = insertelement undef, i8 %b, i8 0 %splat = shufflevector %head, undef, zeroinitializer diff --git a/llvm/test/CodeGen/RISCV/rvv/vwsub-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vwsub-sdnode.ll index 123469ade0ed..852814d648bf 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vwsub-sdnode.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vwsub-sdnode.ll @@ -421,10 +421,10 @@ define @vwsubu_wx_nxv8i64_nxv8i32( %va, i32 define @vwsub_vv_nxv1i64_nxv1i16( %va, %vb) { ; CHECK-LABEL: vwsub_vv_nxv1i64_nxv1i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v8 -; CHECK-NEXT: vsext.vf4 v8, v9 -; CHECK-NEXT: vsub.vv v8, v10, v8 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v8 +; CHECK-NEXT: vsext.vf2 v11, v9 +; CHECK-NEXT: vwsub.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -451,10 +451,10 @@ define @vwsub_vx_nxv1i64_nxv1i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v8 -; CHECK-NEXT: vsext.vf4 v8, v9 -; CHECK-NEXT: vsub.vv v8, v10, v8 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v8 +; CHECK-NEXT: vsext.vf2 v11, v9 +; CHECK-NEXT: vwsub.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -483,9 +483,9 @@ define @vwsubu_vx_nxv1i64_nxv1i16( %va, i16 define @vwsub_wv_nxv1i64_nxv1i16( %va, %vb) { ; CHECK-LABEL: vwsub_wv_nxv1i64_nxv1i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v9 -; CHECK-NEXT: vsub.vv v8, v8, v10 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v9 +; CHECK-NEXT: vwsub.wv v8, v8, v10 ; CHECK-NEXT: ret %vc = sext %vb to %vd = sub %va, %vc @@ -495,9 +495,9 @@ define @vwsub_wv_nxv1i64_nxv1i16( %va, @vwsubu_wv_nxv1i64_nxv1i16( %va, %vb) { ; CHECK-LABEL: vwsubu_wv_nxv1i64_nxv1i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v9 -; CHECK-NEXT: vsub.vv v8, v8, v10 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v9 +; CHECK-NEXT: vwsubu.wv v8, v8, v10 ; CHECK-NEXT: ret %vc = zext %vb to %vd = sub %va, %vc @@ -509,9 +509,9 @@ define @vwsub_wx_nxv1i64_nxv1i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v9 -; CHECK-NEXT: vsub.vv v8, v8, v10 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v9 +; CHECK-NEXT: vwsub.wv v8, v8, v10 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -525,9 +525,9 @@ define @vwsubu_wx_nxv1i64_nxv1i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v9 -; CHECK-NEXT: vsub.vv v8, v8, v10 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v9 +; CHECK-NEXT: vwsubu.wv v8, v8, v10 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -539,10 +539,10 @@ define @vwsubu_wx_nxv1i64_nxv1i16( %va, i16 define @vwsub_vv_nxv2i64_nxv2i16( %va, %vb) { ; CHECK-LABEL: vwsub_vv_nxv2i64_nxv2i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v8 -; CHECK-NEXT: vsext.vf4 v12, v9 -; CHECK-NEXT: vsub.vv v8, v10, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v8 +; CHECK-NEXT: vsext.vf2 v11, v9 +; CHECK-NEXT: vwsub.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -569,10 +569,10 @@ define @vwsub_vx_nxv2i64_nxv2i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v8 -; CHECK-NEXT: vsext.vf4 v12, v9 -; CHECK-NEXT: vsub.vv v8, v10, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v8 +; CHECK-NEXT: vsext.vf2 v11, v9 +; CHECK-NEXT: vwsub.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -601,9 +601,9 @@ define @vwsubu_vx_nxv2i64_nxv2i16( %va, i16 define @vwsub_wv_nxv2i64_nxv2i16( %va, %vb) { ; CHECK-LABEL: vwsub_wv_nxv2i64_nxv2i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf4 v12, v10 -; CHECK-NEXT: vsub.vv v8, v8, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf2 v11, v10 +; CHECK-NEXT: vwsub.wv v8, v8, v11 ; CHECK-NEXT: ret %vc = sext %vb to %vd = sub %va, %vc @@ -613,9 +613,9 @@ define @vwsub_wv_nxv2i64_nxv2i16( %va, @vwsubu_wv_nxv2i64_nxv2i16( %va, %vb) { ; CHECK-LABEL: vwsubu_wv_nxv2i64_nxv2i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vzext.vf4 v12, v10 -; CHECK-NEXT: vsub.vv v8, v8, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vzext.vf2 v11, v10 +; CHECK-NEXT: vwsubu.wv v8, v8, v11 ; CHECK-NEXT: ret %vc = zext %vb to %vd = sub %va, %vc @@ -627,9 +627,9 @@ define @vwsub_wx_nxv2i64_nxv2i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf4 v12, v10 -; CHECK-NEXT: vsub.vv v8, v8, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf2 v11, v10 +; CHECK-NEXT: vwsub.wv v8, v8, v11 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -643,9 +643,9 @@ define @vwsubu_wx_nxv2i64_nxv2i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vzext.vf4 v12, v10 -; CHECK-NEXT: vsub.vv v8, v8, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vzext.vf2 v11, v10 +; CHECK-NEXT: vwsubu.wv v8, v8, v11 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -657,10 +657,10 @@ define @vwsubu_wx_nxv2i64_nxv2i16( %va, i16 define @vwsub_vv_nxv4i64_nxv4i16( %va, %vb) { ; CHECK-LABEL: vwsub_vv_nxv4i64_nxv4i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf4 v12, v8 -; CHECK-NEXT: vsext.vf4 v16, v9 -; CHECK-NEXT: vsub.vv v8, v12, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf2 v12, v8 +; CHECK-NEXT: vsext.vf2 v14, v9 +; CHECK-NEXT: vwsub.vv v8, v12, v14 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -687,10 +687,10 @@ define @vwsub_vx_nxv4i64_nxv4i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf4 v12, v8 -; CHECK-NEXT: vsext.vf4 v16, v9 -; CHECK-NEXT: vsub.vv v8, v12, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf2 v12, v8 +; CHECK-NEXT: vsext.vf2 v14, v9 +; CHECK-NEXT: vwsub.vv v8, v12, v14 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -719,9 +719,9 @@ define @vwsubu_vx_nxv4i64_nxv4i16( %va, i16 define @vwsub_wv_nxv4i64_nxv4i16( %va, %vb) { ; CHECK-LABEL: vwsub_wv_nxv4i64_nxv4i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf4 v16, v12 -; CHECK-NEXT: vsub.vv v8, v8, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf2 v14, v12 +; CHECK-NEXT: vwsub.wv v8, v8, v14 ; CHECK-NEXT: ret %vc = sext %vb to %vd = sub %va, %vc @@ -731,9 +731,9 @@ define @vwsub_wv_nxv4i64_nxv4i16( %va, @vwsubu_wv_nxv4i64_nxv4i16( %va, %vb) { ; CHECK-LABEL: vwsubu_wv_nxv4i64_nxv4i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vzext.vf4 v16, v12 -; CHECK-NEXT: vsub.vv v8, v8, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v14, v12 +; CHECK-NEXT: vwsubu.wv v8, v8, v14 ; CHECK-NEXT: ret %vc = zext %vb to %vd = sub %va, %vc @@ -745,9 +745,9 @@ define @vwsub_wx_nxv4i64_nxv4i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma ; CHECK-NEXT: vmv.v.x v12, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf4 v16, v12 -; CHECK-NEXT: vsub.vv v8, v8, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf2 v14, v12 +; CHECK-NEXT: vwsub.wv v8, v8, v14 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -761,9 +761,9 @@ define @vwsubu_wx_nxv4i64_nxv4i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma ; CHECK-NEXT: vmv.v.x v12, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vzext.vf4 v16, v12 -; CHECK-NEXT: vsub.vv v8, v8, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v14, v12 +; CHECK-NEXT: vwsubu.wv v8, v8, v14 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -775,10 +775,10 @@ define @vwsubu_wx_nxv4i64_nxv4i16( %va, i16 define @vwsub_vv_nxv8i64_nxv8i16( %va, %vb) { ; CHECK-LABEL: vwsub_vv_nxv8i64_nxv8i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf4 v16, v8 -; CHECK-NEXT: vsext.vf4 v24, v10 -; CHECK-NEXT: vsub.vv v8, v16, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf2 v16, v8 +; CHECK-NEXT: vsext.vf2 v20, v10 +; CHECK-NEXT: vwsub.vv v8, v16, v20 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -805,10 +805,10 @@ define @vwsub_vx_nxv8i64_nxv8i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma ; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf4 v16, v8 -; CHECK-NEXT: vsext.vf4 v24, v10 -; CHECK-NEXT: vsub.vv v8, v16, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf2 v16, v8 +; CHECK-NEXT: vsext.vf2 v20, v10 +; CHECK-NEXT: vwsub.vv v8, v16, v20 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -837,9 +837,9 @@ define @vwsubu_vx_nxv8i64_nxv8i16( %va, i16 define @vwsub_wv_nxv8i64_nxv8i16( %va, %vb) { ; CHECK-LABEL: vwsub_wv_nxv8i64_nxv8i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf4 v24, v16 -; CHECK-NEXT: vsub.vv v8, v8, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf2 v20, v16 +; CHECK-NEXT: vwsub.wv v8, v8, v20 ; CHECK-NEXT: ret %vc = sext %vb to %vd = sub %va, %vc @@ -849,9 +849,9 @@ define @vwsub_wv_nxv8i64_nxv8i16( %va, @vwsubu_wv_nxv8i64_nxv8i16( %va, %vb) { ; CHECK-LABEL: vwsubu_wv_nxv8i64_nxv8i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vzext.vf4 v24, v16 -; CHECK-NEXT: vsub.vv v8, v8, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vzext.vf2 v20, v16 +; CHECK-NEXT: vwsubu.wv v8, v8, v20 ; CHECK-NEXT: ret %vc = zext %vb to %vd = sub %va, %vc @@ -863,9 +863,9 @@ define @vwsub_wx_nxv8i64_nxv8i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma ; CHECK-NEXT: vmv.v.x v16, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf4 v24, v16 -; CHECK-NEXT: vsub.vv v8, v8, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf2 v20, v16 +; CHECK-NEXT: vwsub.wv v8, v8, v20 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -879,9 +879,9 @@ define @vwsubu_wx_nxv8i64_nxv8i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma ; CHECK-NEXT: vmv.v.x v16, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vzext.vf4 v24, v16 -; CHECK-NEXT: vsub.vv v8, v8, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vzext.vf2 v20, v16 +; CHECK-NEXT: vwsubu.wv v8, v8, v20 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -893,10 +893,10 @@ define @vwsubu_wx_nxv8i64_nxv8i16( %va, i16 define @vwsub_vv_nxv1i64_nxv1i8( %va, %vb) { ; CHECK-LABEL: vwsub_vv_nxv1i64_nxv1i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v8 -; CHECK-NEXT: vsext.vf8 v8, v9 -; CHECK-NEXT: vsub.vv v8, v10, v8 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vsext.vf4 v11, v9 +; CHECK-NEXT: vwsub.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -923,10 +923,10 @@ define @vwsub_vx_nxv1i64_nxv1i8( %va, i8 %b) ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v8 -; CHECK-NEXT: vsext.vf8 v8, v9 -; CHECK-NEXT: vsub.vv v8, v10, v8 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vsext.vf4 v11, v9 +; CHECK-NEXT: vwsub.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -955,9 +955,9 @@ define @vwsubu_vx_nxv1i64_nxv1i8( %va, i8 %b define @vwsub_wv_nxv1i64_nxv1i8( %va, %vb) { ; CHECK-LABEL: vwsub_wv_nxv1i64_nxv1i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v9 -; CHECK-NEXT: vsub.vv v8, v8, v10 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v9 +; CHECK-NEXT: vwsub.wv v8, v8, v10 ; CHECK-NEXT: ret %vc = sext %vb to %vd = sub %va, %vc @@ -967,9 +967,9 @@ define @vwsub_wv_nxv1i64_nxv1i8( %va, @vwsubu_wv_nxv1i64_nxv1i8( %va, %vb) { ; CHECK-LABEL: vwsubu_wv_nxv1i64_nxv1i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vzext.vf8 v10, v9 -; CHECK-NEXT: vsub.vv v8, v8, v10 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf4 v10, v9 +; CHECK-NEXT: vwsubu.wv v8, v8, v10 ; CHECK-NEXT: ret %vc = zext %vb to %vd = sub %va, %vc @@ -981,9 +981,9 @@ define @vwsub_wx_nxv1i64_nxv1i8( %va, i8 %b ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v9 -; CHECK-NEXT: vsub.vv v8, v8, v10 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v9 +; CHECK-NEXT: vwsub.wv v8, v8, v10 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -997,9 +997,9 @@ define @vwsubu_wx_nxv1i64_nxv1i8( %va, i8 % ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vzext.vf8 v10, v9 -; CHECK-NEXT: vsub.vv v8, v8, v10 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf4 v10, v9 +; CHECK-NEXT: vwsubu.wv v8, v8, v10 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1011,10 +1011,10 @@ define @vwsubu_wx_nxv1i64_nxv1i8( %va, i8 % define @vwsub_vv_nxv2i64_nxv2i8( %va, %vb) { ; CHECK-LABEL: vwsub_vv_nxv2i64_nxv2i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v8 -; CHECK-NEXT: vsext.vf8 v12, v9 -; CHECK-NEXT: vsub.vv v8, v10, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vsext.vf4 v11, v9 +; CHECK-NEXT: vwsub.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -1041,10 +1041,10 @@ define @vwsub_vx_nxv2i64_nxv2i8( %va, i8 %b) ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v8 -; CHECK-NEXT: vsext.vf8 v12, v9 -; CHECK-NEXT: vsub.vv v8, v10, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vsext.vf4 v11, v9 +; CHECK-NEXT: vwsub.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1073,9 +1073,9 @@ define @vwsubu_vx_nxv2i64_nxv2i8( %va, i8 %b define @vwsub_wv_nxv2i64_nxv2i8( %va, %vb) { ; CHECK-LABEL: vwsub_wv_nxv2i64_nxv2i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf8 v12, v10 -; CHECK-NEXT: vsub.vv v8, v8, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf4 v11, v10 +; CHECK-NEXT: vwsub.wv v8, v8, v11 ; CHECK-NEXT: ret %vc = sext %vb to %vd = sub %va, %vc @@ -1085,9 +1085,9 @@ define @vwsub_wv_nxv2i64_nxv2i8( %va, @vwsubu_wv_nxv2i64_nxv2i8( %va, %vb) { ; CHECK-LABEL: vwsubu_wv_nxv2i64_nxv2i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vzext.vf8 v12, v10 -; CHECK-NEXT: vsub.vv v8, v8, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vzext.vf4 v11, v10 +; CHECK-NEXT: vwsubu.wv v8, v8, v11 ; CHECK-NEXT: ret %vc = zext %vb to %vd = sub %va, %vc @@ -1099,9 +1099,9 @@ define @vwsub_wx_nxv2i64_nxv2i8( %va, i8 %b ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf8 v12, v10 -; CHECK-NEXT: vsub.vv v8, v8, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf4 v11, v10 +; CHECK-NEXT: vwsub.wv v8, v8, v11 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1115,9 +1115,9 @@ define @vwsubu_wx_nxv2i64_nxv2i8( %va, i8 % ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vzext.vf8 v12, v10 -; CHECK-NEXT: vsub.vv v8, v8, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vzext.vf4 v11, v10 +; CHECK-NEXT: vwsubu.wv v8, v8, v11 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1129,10 +1129,10 @@ define @vwsubu_wx_nxv2i64_nxv2i8( %va, i8 % define @vwsub_vv_nxv4i64_nxv4i8( %va, %vb) { ; CHECK-LABEL: vwsub_vv_nxv4i64_nxv4i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf8 v12, v8 -; CHECK-NEXT: vsext.vf8 v16, v9 -; CHECK-NEXT: vsub.vv v8, v12, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf4 v12, v8 +; CHECK-NEXT: vsext.vf4 v14, v9 +; CHECK-NEXT: vwsub.vv v8, v12, v14 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -1159,10 +1159,10 @@ define @vwsub_vx_nxv4i64_nxv4i8( %va, i8 %b) ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf8 v12, v8 -; CHECK-NEXT: vsext.vf8 v16, v9 -; CHECK-NEXT: vsub.vv v8, v12, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf4 v12, v8 +; CHECK-NEXT: vsext.vf4 v14, v9 +; CHECK-NEXT: vwsub.vv v8, v12, v14 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1191,9 +1191,9 @@ define @vwsubu_vx_nxv4i64_nxv4i8( %va, i8 %b define @vwsub_wv_nxv4i64_nxv4i8( %va, %vb) { ; CHECK-LABEL: vwsub_wv_nxv4i64_nxv4i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf8 v16, v12 -; CHECK-NEXT: vsub.vv v8, v8, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf4 v14, v12 +; CHECK-NEXT: vwsub.wv v8, v8, v14 ; CHECK-NEXT: ret %vc = sext %vb to %vd = sub %va, %vc @@ -1203,9 +1203,9 @@ define @vwsub_wv_nxv4i64_nxv4i8( %va, @vwsubu_wv_nxv4i64_nxv4i8( %va, %vb) { ; CHECK-LABEL: vwsubu_wv_nxv4i64_nxv4i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vzext.vf8 v16, v12 -; CHECK-NEXT: vsub.vv v8, v8, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf4 v14, v12 +; CHECK-NEXT: vwsubu.wv v8, v8, v14 ; CHECK-NEXT: ret %vc = zext %vb to %vd = sub %va, %vc @@ -1217,9 +1217,9 @@ define @vwsub_wx_nxv4i64_nxv4i8( %va, i8 %b ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v12, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf8 v16, v12 -; CHECK-NEXT: vsub.vv v8, v8, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf4 v14, v12 +; CHECK-NEXT: vwsub.wv v8, v8, v14 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1233,9 +1233,9 @@ define @vwsubu_wx_nxv4i64_nxv4i8( %va, i8 % ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v12, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vzext.vf8 v16, v12 -; CHECK-NEXT: vsub.vv v8, v8, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf4 v14, v12 +; CHECK-NEXT: vwsubu.wv v8, v8, v14 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1247,10 +1247,10 @@ define @vwsubu_wx_nxv4i64_nxv4i8( %va, i8 % define @vwsub_vv_nxv8i64_nxv8i8( %va, %vb) { ; CHECK-LABEL: vwsub_vv_nxv8i64_nxv8i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vsext.vf8 v24, v9 -; CHECK-NEXT: vsub.vv v8, v16, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf4 v16, v8 +; CHECK-NEXT: vsext.vf4 v20, v9 +; CHECK-NEXT: vwsub.vv v8, v16, v20 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -1277,10 +1277,10 @@ define @vwsub_vx_nxv8i64_nxv8i8( %va, i8 %b) ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vsext.vf8 v24, v9 -; CHECK-NEXT: vsub.vv v8, v16, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf4 v16, v8 +; CHECK-NEXT: vsext.vf4 v20, v9 +; CHECK-NEXT: vwsub.vv v8, v16, v20 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1309,9 +1309,9 @@ define @vwsubu_vx_nxv8i64_nxv8i8( %va, i8 %b define @vwsub_wv_nxv8i64_nxv8i8( %va, %vb) { ; CHECK-LABEL: vwsub_wv_nxv8i64_nxv8i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf8 v24, v16 -; CHECK-NEXT: vsub.vv v8, v8, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf4 v20, v16 +; CHECK-NEXT: vwsub.wv v8, v8, v20 ; CHECK-NEXT: ret %vc = sext %vb to %vd = sub %va, %vc @@ -1321,9 +1321,9 @@ define @vwsub_wv_nxv8i64_nxv8i8( %va, @vwsubu_wv_nxv8i64_nxv8i8( %va, %vb) { ; CHECK-LABEL: vwsubu_wv_nxv8i64_nxv8i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vzext.vf8 v24, v16 -; CHECK-NEXT: vsub.vv v8, v8, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vzext.vf4 v20, v16 +; CHECK-NEXT: vwsubu.wv v8, v8, v20 ; CHECK-NEXT: ret %vc = zext %vb to %vd = sub %va, %vc @@ -1335,9 +1335,9 @@ define @vwsub_wx_nxv8i64_nxv8i8( %va, i8 %b ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma ; CHECK-NEXT: vmv.v.x v16, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf8 v24, v16 -; CHECK-NEXT: vsub.vv v8, v8, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf4 v20, v16 +; CHECK-NEXT: vwsub.wv v8, v8, v20 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1351,9 +1351,9 @@ define @vwsubu_wx_nxv8i64_nxv8i8( %va, i8 % ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma ; CHECK-NEXT: vmv.v.x v16, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vzext.vf8 v24, v16 -; CHECK-NEXT: vsub.vv v8, v8, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vzext.vf4 v20, v16 +; CHECK-NEXT: vwsubu.wv v8, v8, v20 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer -- GitLab From d22cf4365ca58ccf1db21341d63ac49921f1c47a Mon Sep 17 00:00:00 2001 From: Nathan Lanza Date: Thu, 21 Mar 2024 19:52:19 -0400 Subject: [PATCH 011/306] [cmake] Place clang behind mlir in the liist of external projects In preparation for the initial ClangIR upstreaming process, move clang behind MLIR in the list of external projects. Otherwise, cmake will attempt to build clang before MLIR. reland of https://github.com/llvm/llvm-project/pull/86050 Reviewers: Pull Request: https://github.com/llvm/llvm-project/pull/86210 --- llvm/tools/CMakeLists.txt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/llvm/tools/CMakeLists.txt b/llvm/tools/CMakeLists.txt index c6116ac81d12..cde57367934e 100644 --- a/llvm/tools/CMakeLists.txt +++ b/llvm/tools/CMakeLists.txt @@ -37,12 +37,13 @@ add_llvm_tool_subdirectory(llvm-profdata) # Projects supported via LLVM_EXTERNAL_*_SOURCE_DIR need to be explicitly # specified. -add_llvm_external_project(clang) add_llvm_external_project(lld) -add_llvm_external_project(lldb) add_llvm_external_project(mlir) -# Flang depends on mlir, so place it afterward +# ClangIR and Flang depend on mlir, lldb and Flang depend on clang, sort them +# accordingly so place them afterwards +add_llvm_external_project(clang) add_llvm_external_project(flang) +add_llvm_external_project(lldb) add_llvm_external_project(bolt) # Automatically add remaining sub-directories containing a 'CMakeLists.txt' -- GitLab From e66b670f3bf9312f696e66c31152ae535207d6bb Mon Sep 17 00:00:00 2001 From: Nathan Lanza Date: Thu, 21 Mar 2024 19:53:48 -0400 Subject: [PATCH 012/306] [CIR][Basic][NFC] Add the CIR language to the Language enum Add the CIR language to the Language enum and the standard usages of it. commit-id:fd12b2c2 Reviewers: bcardosolopes, AaronBallman, erichkeane Reviewed By: AaronBallman, bcardosolopes Pull Request: https://github.com/llvm/llvm-project/pull/86072 --- clang/include/clang/Basic/LangStandard.h | 5 +++-- clang/include/clang/Driver/Types.def | 1 + clang/lib/Basic/LangStandards.cpp | 3 +++ .../Serialization/SymbolGraphSerializer.cpp | 1 + clang/lib/Frontend/CompilerInvocation.cpp | 13 +++++++++++-- clang/lib/Frontend/FrontendActions.cpp | 1 + clang/lib/Frontend/FrontendOptions.cpp | 1 + 7 files changed, 21 insertions(+), 4 deletions(-) diff --git a/clang/include/clang/Basic/LangStandard.h b/clang/include/clang/Basic/LangStandard.h index 199e24c67316..8e25afc83366 100644 --- a/clang/include/clang/Basic/LangStandard.h +++ b/clang/include/clang/Basic/LangStandard.h @@ -26,8 +26,9 @@ enum class Language : uint8_t { /// Assembly: we accept this only so that we can preprocess it. Asm, - /// LLVM IR: we accept this so that we can run the optimizer on it, - /// and compile it to assembly or object code. + /// LLVM IR & CIR: we accept these so that we can run the optimizer on them, + /// and compile them to assembly or object code (or LLVM for CIR). + CIR, LLVM_IR, ///@{ Languages that the frontend can parse and compile. diff --git a/clang/include/clang/Driver/Types.def b/clang/include/clang/Driver/Types.def index f72c27e1ee70..0e0cae5fb706 100644 --- a/clang/include/clang/Driver/Types.def +++ b/clang/include/clang/Driver/Types.def @@ -90,6 +90,7 @@ TYPE("ir", LLVM_BC, INVALID, "bc", phases TYPE("lto-ir", LTO_IR, INVALID, "s", phases::Compile, phases::Backend, phases::Assemble, phases::Link) TYPE("lto-bc", LTO_BC, INVALID, "o", phases::Compile, phases::Backend, phases::Assemble, phases::Link) +TYPE("cir", CIR, INVALID, "cir", phases::Compile, phases::Backend, phases::Assemble, phases::Link) // Misc. TYPE("ast", AST, INVALID, "ast", phases::Compile, phases::Backend, phases::Assemble, phases::Link) TYPE("ifs", IFS, INVALID, "ifs", phases::IfsMerge) diff --git a/clang/lib/Basic/LangStandards.cpp b/clang/lib/Basic/LangStandards.cpp index cb2c07723499..c8c9292abcb2 100644 --- a/clang/lib/Basic/LangStandards.cpp +++ b/clang/lib/Basic/LangStandards.cpp @@ -21,6 +21,8 @@ StringRef clang::languageToString(Language L) { return "Asm"; case Language::LLVM_IR: return "LLVM IR"; + case Language::CIR: + return "ClangIR"; case Language::C: return "C"; case Language::CXX: @@ -92,6 +94,7 @@ LangStandard::Kind clang::getDefaultLanguageStandard(clang::Language Lang, switch (Lang) { case Language::Unknown: case Language::LLVM_IR: + case Language::CIR: llvm_unreachable("Invalid input kind!"); case Language::OpenCL: return LangStandard::lang_opencl12; diff --git a/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp b/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp index 349b93e2a232..545860acb7db 100644 --- a/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp +++ b/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp @@ -208,6 +208,7 @@ StringRef getLanguageName(Language Lang) { case Language::Unknown: case Language::Asm: case Language::LLVM_IR: + case Language::CIR: llvm_unreachable("Unsupported language kind"); } diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp index 0df6a82ccd89..7bd91d4791ec 100644 --- a/clang/lib/Frontend/CompilerInvocation.cpp +++ b/clang/lib/Frontend/CompilerInvocation.cpp @@ -2757,6 +2757,9 @@ static void GenerateFrontendArgs(const FrontendOptions &Opts, case Language::HLSL: Lang = "hlsl"; break; + case Language::CIR: + Lang = "cir"; + break; } GenerateArg(Consumer, OPT_x, @@ -2958,6 +2961,7 @@ static bool ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args, .Cases("ast", "pcm", "precompiled-header", InputKind(Language::Unknown, InputKind::Precompiled)) .Case("ir", Language::LLVM_IR) + .Case("cir", Language::CIR) .Default(Language::Unknown); if (DashX.isUnknown()) @@ -3323,6 +3327,7 @@ static bool IsInputCompatibleWithStandard(InputKind IK, switch (IK.getLanguage()) { case Language::Unknown: case Language::LLVM_IR: + case Language::CIR: llvm_unreachable("should not parse language flags for this input"); case Language::C: @@ -3388,6 +3393,8 @@ static StringRef GetInputKindName(InputKind IK) { return "Asm"; case Language::LLVM_IR: return "LLVM IR"; + case Language::CIR: + return "Clang IR"; case Language::HLSL: return "HLSL"; @@ -3403,7 +3410,8 @@ void CompilerInvocationBase::GenerateLangArgs(const LangOptions &Opts, const llvm::Triple &T, InputKind IK) { if (IK.getFormat() == InputKind::Precompiled || - IK.getLanguage() == Language::LLVM_IR) { + IK.getLanguage() == Language::LLVM_IR || + IK.getLanguage() == Language::CIR) { if (Opts.ObjCAutoRefCount) GenerateArg(Consumer, OPT_fobjc_arc); if (Opts.PICLevel != 0) @@ -3689,7 +3697,8 @@ bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args, unsigned NumErrorsBefore = Diags.getNumErrors(); if (IK.getFormat() == InputKind::Precompiled || - IK.getLanguage() == Language::LLVM_IR) { + IK.getLanguage() == Language::LLVM_IR || + IK.getLanguage() == Language::CIR) { // ObjCAAutoRefCount and Sanitize LangOpts are used to setup the // PassManager in BackendUtil.cpp. They need to be initialized no matter // what the input type is. diff --git a/clang/lib/Frontend/FrontendActions.cpp b/clang/lib/Frontend/FrontendActions.cpp index 81fcd8d5ae9b..3fd1cdd3b479 100644 --- a/clang/lib/Frontend/FrontendActions.cpp +++ b/clang/lib/Frontend/FrontendActions.cpp @@ -1083,6 +1083,7 @@ void PrintPreambleAction::ExecuteAction() { case Language::CUDA: case Language::HIP: case Language::HLSL: + case Language::CIR: break; case Language::Unknown: diff --git a/clang/lib/Frontend/FrontendOptions.cpp b/clang/lib/Frontend/FrontendOptions.cpp index bf83b27c1367..32ed99571e85 100644 --- a/clang/lib/Frontend/FrontendOptions.cpp +++ b/clang/lib/Frontend/FrontendOptions.cpp @@ -34,5 +34,6 @@ InputKind FrontendOptions::getInputKindForExtension(StringRef Extension) { .Case("hip", Language::HIP) .Cases("ll", "bc", Language::LLVM_IR) .Case("hlsl", Language::HLSL) + .Case("cir", Language::CIR) .Default(Language::Unknown); } -- GitLab From 3942bd2fb56380aa050977dc6aede011e191d9b0 Mon Sep 17 00:00:00 2001 From: Alexey Bataev Date: Thu, 21 Mar 2024 17:05:50 -0700 Subject: [PATCH 013/306] [SLP]Fix a crash if the argument of call was affected by minbitwidt analysis. Need to support proper type conversion for function arguments to avoid compiler crash. --- .../Transforms/Vectorize/SLPVectorizer.cpp | 9 +- .../X86/call-arg-reduced-by-minbitwidth.ll | 82 +++++++++++++++++++ 2 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/call-arg-reduced-by-minbitwidth.ll diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 36b446962c4a..7295ae0ba90b 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -12473,12 +12473,12 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E, bool PostponedPHIs) { if (UseIntrinsic && isVectorIntrinsicWithOverloadTypeAtArg(ID, -1)) TysForDecl.push_back( FixedVectorType::get(CI->getType(), E->Scalars.size())); + auto *CEI = cast(VL0); for (unsigned I : seq(0, CI->arg_size())) { ValueList OpVL; // Some intrinsics have scalar arguments. This argument should not be // vectorized. if (UseIntrinsic && isVectorIntrinsicWithScalarOpAtArg(ID, I)) { - CallInst *CEI = cast(VL0); ScalarArg = CEI->getArgOperand(I); OpVecs.push_back(CEI->getArgOperand(I)); if (isVectorIntrinsicWithOverloadTypeAtArg(ID, I)) @@ -12491,6 +12491,13 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E, bool PostponedPHIs) { LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n"); return E->VectorizedValue; } + ScalarArg = CEI->getArgOperand(I); + if (cast(OpVec->getType())->getElementType() != + ScalarArg->getType()) { + auto *CastTy = FixedVectorType::get(ScalarArg->getType(), + VecTy->getNumElements()); + OpVec = Builder.CreateIntCast(OpVec, CastTy, GetOperandSignedness(I)); + } LLVM_DEBUG(dbgs() << "SLP: OpVec[" << I << "]: " << *OpVec << "\n"); OpVecs.push_back(OpVec); if (UseIntrinsic && isVectorIntrinsicWithOverloadTypeAtArg(ID, I)) diff --git a/llvm/test/Transforms/SLPVectorizer/X86/call-arg-reduced-by-minbitwidth.ll b/llvm/test/Transforms/SLPVectorizer/X86/call-arg-reduced-by-minbitwidth.ll new file mode 100644 index 000000000000..27c9655f94d3 --- /dev/null +++ b/llvm/test/Transforms/SLPVectorizer/X86/call-arg-reduced-by-minbitwidth.ll @@ -0,0 +1,82 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-pc-windows-msvc19.34.0 < %s | FileCheck %s + +define void @test(ptr %0, i8 %1, i1 %cmp12.i) { +; CHECK-LABEL: define void @test( +; CHECK-SAME: ptr [[TMP0:%.*]], i8 [[TMP1:%.*]], i1 [[CMP12_I:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP2:%.*]] = insertelement <8 x i1> poison, i1 [[CMP12_I]], i32 0 +; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> poison, <8 x i32> zeroinitializer +; CHECK-NEXT: [[TMP4:%.*]] = insertelement <8 x i8> poison, i8 [[TMP1]], i32 0 +; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <8 x i8> [[TMP4]], <8 x i8> poison, <8 x i32> zeroinitializer +; CHECK-NEXT: br label [[PRE:%.*]] +; CHECK: pre: +; CHECK-NEXT: [[TMP6:%.*]] = zext <8 x i8> [[TMP5]] to <8 x i32> +; CHECK-NEXT: [[TMP7:%.*]] = call <8 x i32> @llvm.umax.v8i32(<8 x i32> [[TMP6]], <8 x i32> ) +; CHECK-NEXT: [[TMP8:%.*]] = trunc <8 x i32> [[TMP7]] to <8 x i8> +; CHECK-NEXT: [[TMP9:%.*]] = add <8 x i8> [[TMP8]], +; CHECK-NEXT: [[TMP10:%.*]] = select <8 x i1> [[TMP3]], <8 x i8> [[TMP9]], <8 x i8> [[TMP5]] +; CHECK-NEXT: store <8 x i8> [[TMP10]], ptr [[TMP0]], align 1 +; CHECK-NEXT: br label [[PRE]] +; +entry: + %idx11 = getelementptr i8, ptr %0, i64 1 + %idx22 = getelementptr i8, ptr %0, i64 2 + %idx33 = getelementptr i8, ptr %0, i64 3 + %idx44 = getelementptr i8, ptr %0, i64 4 + %idx55 = getelementptr i8, ptr %0, i64 5 + %idx66 = getelementptr i8, ptr %0, i64 6 + %idx77 = getelementptr i8, ptr %0, i64 7 + br label %pre + +pre: + %conv.i = zext i8 %1 to i32 + %2 = tail call i32 @llvm.umax.i32(i32 %conv.i, i32 1) + %.sroa.speculated.i = add i32 %2, 1 + %intensity.0.i = select i1 %cmp12.i, i32 %.sroa.speculated.i, i32 %conv.i + %conv14.i = trunc i32 %intensity.0.i to i8 + store i8 %conv14.i, ptr %0, align 1 + %conv.i.1 = zext i8 %1 to i32 + %3 = tail call i32 @llvm.umax.i32(i32 %conv.i.1, i32 1) + %ss1 = add i32 %3, 1 + %ii1 = select i1 %cmp12.i, i32 %ss1, i32 %conv.i.1 + %conv14.i.1 = trunc i32 %ii1 to i8 + store i8 %conv14.i.1, ptr %idx11, align 1 + %conv.i.2 = zext i8 %1 to i32 + %4 = tail call i32 @llvm.umax.i32(i32 %conv.i.2, i32 1) + %ss2 = add i32 %4, 1 + %ii2 = select i1 %cmp12.i, i32 %ss2, i32 %conv.i.2 + %conv14.i.2 = trunc i32 %ii2 to i8 + store i8 %conv14.i.2, ptr %idx22, align 1 + %conv.i.3 = zext i8 %1 to i32 + %5 = tail call i32 @llvm.umax.i32(i32 %conv.i.3, i32 1) + %ss3 = add i32 %5, 1 + %ii3 = select i1 %cmp12.i, i32 %ss3, i32 %conv.i.3 + %conv14.i.3 = trunc i32 %ii3 to i8 + store i8 %conv14.i.3, ptr %idx33, align 1 + %conv.i.4 = zext i8 %1 to i32 + %6 = tail call i32 @llvm.umax.i32(i32 %conv.i.4, i32 1) + %ss4 = add i32 %6, 1 + %ii4 = select i1 %cmp12.i, i32 %ss4, i32 %conv.i.4 + %conv14.i.4 = trunc i32 %ii4 to i8 + store i8 %conv14.i.4, ptr %idx44, align 1 + %conv.i.5 = zext i8 %1 to i32 + %7 = tail call i32 @llvm.umax.i32(i32 %conv.i.5, i32 1) + %ss5 = add i32 %7, 1 + %ii5 = select i1 %cmp12.i, i32 %ss5, i32 %conv.i.5 + %conv14.i.5 = trunc i32 %ii5 to i8 + store i8 %conv14.i.5, ptr %idx55, align 1 + %conv.i.6 = zext i8 %1 to i32 + %8 = tail call i32 @llvm.umax.i32(i32 %conv.i.6, i32 1) + %ss6 = add i32 %8, 1 + %ii6 = select i1 %cmp12.i, i32 %ss6, i32 %conv.i.6 + %conv14.i.6 = trunc i32 %ii6 to i8 + store i8 %conv14.i.6, ptr %idx66, align 1 + %conv.i.7 = zext i8 %1 to i32 + %9 = tail call i32 @llvm.umax.i32(i32 %conv.i.7, i32 1) + %ss7 = add i32 %9, 1 + %ii7 = select i1 %cmp12.i, i32 %ss7, i32 %conv.i.7 + %conv14.i.7 = trunc i32 %ii7 to i8 + store i8 %conv14.i.7, ptr %idx77, align 1 + br label %pre +} -- GitLab From 7564566779eb07e9daf41a351b09cf7607871845 Mon Sep 17 00:00:00 2001 From: Jonas Paulsson Date: Wed, 20 Mar 2024 11:58:46 -0400 Subject: [PATCH 014/306] Reapply "Move assertion for AdjustsStack from PEI to MachineVerifier (#85698)" - The check is now actually done in both PEI and the MachineVerifier. - More .mir tests trivially updated with "adjustsStack: true" as needed. --- llvm/lib/CodeGen/MachineVerifier.cpp | 6 +++++ .../clear-dead-implicit-def-impdef.mir | 2 ++ ...plicit-def-remat-requires-impdef-check.mir | 2 ++ ...implicit-def-with-impdef-greedy-assert.mir | 2 ++ .../CodeGen/AMDGPU/fold-restore-undef-use.mir | 2 ++ .../greedy-alloc-fail-sgpr1024-spill.mir | 1 + .../ran-out-of-sgprs-allocation-failure.mir | 1 + .../CodeGen/AMDGPU/sched-crash-dbg-value.mir | 2 ++ .../AMDGPU/sgpr-spill-wrong-stack-id.mir | 1 + .../AMDGPU/snippet-copy-bundle-regression.mir | 1 + .../virtregrewrite-undef-identity-copy.mir | 1 + ...no-register-coalescing-in-returnsTwice.mir | 2 ++ .../CodeGen/Hexagon/regalloc-bad-undef.mir | 2 +- .../SystemZ/RAbasic-invalid-LR-update.mir | 2 ++ .../SystemZ/clear-liverange-spillreg.mir | 1 + llvm/test/CodeGen/SystemZ/int-cmp-56.mir | 4 +++ .../SystemZ/regcoal-subranges-update.mir | 2 ++ llvm/test/CodeGen/X86/callbr-asm-kill.mir | 1 + llvm/test/CodeGen/X86/late-remat-update.mir | 1 + llvm/test/CodeGen/X86/limit-split-cost.mir | 1 + llvm/test/CodeGen/X86/regalloc-copy-hints.mir | 1 + .../CodeGen/X86/statepoint-fastregalloc.mir | 4 +++ .../X86/statepoint-invoke-ra-enter-at-end.mir | 2 +- .../X86/statepoint-invoke-ra-hoist-copies.mir | 2 +- .../statepoint-invoke-ra-inline-spiller.mir | 2 +- ...tatepoint-invoke-ra-remove-back-copies.mir | 2 +- .../test/CodeGen/X86/statepoint-invoke-ra.mir | 2 +- .../CodeGen/X86/statepoint-vreg-folding.mir | 2 +- .../memory-operand-folding-tieddef.mir | 1 + .../InstrRef/memory-operand-load-folding.mir | 1 + .../MIR/InstrRef/phi-coalesce-subreg.mir | 1 + .../DebugInfo/MIR/InstrRef/phi-coalescing.mir | 1 + .../MIR/InstrRef/phi-on-stack-coalesced.mir | 1 + .../MIR/InstrRef/phi-on-stack-coalesced2.mir | 1 + .../MIR/InstrRef/phi-regallocd-to-stack.mir | 1 + .../MIR/InstrRef/phi-through-regalloc.mir | 1 + .../MIR/InstrRef/stack-coloring-dbg-phi.mir | 1 + .../MIR/InstrRef/survives-livedebugvars.mir | 1 + .../Mips/livedebugvars-stop-trimming-loc.mir | 2 ++ .../live-debug-vars-unused-arg-debugonly.mir | 2 +- .../MIR/X86/live-debug-vars-unused-arg.mir | 2 +- .../X86/livedebugvars-crossbb-interval.mir | 1 + .../X86/live-debug-vars-intervals.mir | 4 +++ .../MachineVerifier/test_adjustsstack.mir | 26 +++++++++++++++++++ 44 files changed, 92 insertions(+), 9 deletions(-) create mode 100644 llvm/test/MachineVerifier/test_adjustsstack.mir diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp index c69d36fc7fdd..005efe48ac0c 100644 --- a/llvm/lib/CodeGen/MachineVerifier.cpp +++ b/llvm/lib/CodeGen/MachineVerifier.cpp @@ -3697,6 +3697,9 @@ void MachineVerifier::verifyStackFrame() { if (I.getOpcode() == FrameSetupOpcode) { if (BBState.ExitIsSetup) report("FrameSetup is after another FrameSetup", &I); + if (!MRI->isSSA() && !MF->getFrameInfo().adjustsStack()) + report("AdjustsStack not set in presence of a frame pseudo " + "instruction.", &I); BBState.ExitValue -= TII->getFrameTotalSize(I); BBState.ExitIsSetup = true; } @@ -3712,6 +3715,9 @@ void MachineVerifier::verifyStackFrame() { errs() << "FrameDestroy <" << Size << "> is after FrameSetup <" << AbsSPAdj << ">.\n"; } + if (!MRI->isSSA() && !MF->getFrameInfo().adjustsStack()) + report("AdjustsStack not set in presence of a frame pseudo " + "instruction.", &I); BBState.ExitValue += Size; BBState.ExitIsSetup = false; } diff --git a/llvm/test/CodeGen/AArch64/clear-dead-implicit-def-impdef.mir b/llvm/test/CodeGen/AArch64/clear-dead-implicit-def-impdef.mir index 9040937d027d..1592c86e267f 100644 --- a/llvm/test/CodeGen/AArch64/clear-dead-implicit-def-impdef.mir +++ b/llvm/test/CodeGen/AArch64/clear-dead-implicit-def-impdef.mir @@ -3,6 +3,8 @@ --- name: func tracksRegLiveness: true +frameInfo: + adjustsStack: true body: | bb.0: liveins: $x0, $x1, $x2, $x3, $x4, $x5, $x6 diff --git a/llvm/test/CodeGen/AArch64/implicit-def-remat-requires-impdef-check.mir b/llvm/test/CodeGen/AArch64/implicit-def-remat-requires-impdef-check.mir index aa94a03786f5..47aa34e3c011 100644 --- a/llvm/test/CodeGen/AArch64/implicit-def-remat-requires-impdef-check.mir +++ b/llvm/test/CodeGen/AArch64/implicit-def-remat-requires-impdef-check.mir @@ -22,6 +22,7 @@ name: inst_stores_to_dead_spill_implicit_def_impdef tracksRegLiveness: true frameInfo: + adjustsStack: true hasCalls: true body: | bb.0: @@ -59,6 +60,7 @@ body: | name: inst_stores_to_dead_spill_movimm_impdef tracksRegLiveness: true frameInfo: + adjustsStack: true hasCalls: true body: | bb.0: diff --git a/llvm/test/CodeGen/AArch64/implicit-def-with-impdef-greedy-assert.mir b/llvm/test/CodeGen/AArch64/implicit-def-with-impdef-greedy-assert.mir index e5395b20afd4..a5d74ef75f0a 100644 --- a/llvm/test/CodeGen/AArch64/implicit-def-with-impdef-greedy-assert.mir +++ b/llvm/test/CodeGen/AArch64/implicit-def-with-impdef-greedy-assert.mir @@ -4,6 +4,8 @@ --- name: widget tracksRegLiveness: true +frameInfo: + adjustsStack: true jumpTable: kind: label-difference32 entries: diff --git a/llvm/test/CodeGen/AMDGPU/fold-restore-undef-use.mir b/llvm/test/CodeGen/AMDGPU/fold-restore-undef-use.mir index 3616d617f84a..5ef8a94eeaa7 100644 --- a/llvm/test/CodeGen/AMDGPU/fold-restore-undef-use.mir +++ b/llvm/test/CodeGen/AMDGPU/fold-restore-undef-use.mir @@ -8,6 +8,8 @@ --- name: restore_undef_copy_use tracksRegLiveness: true +frameInfo: + adjustsStack: true machineFunctionInfo: maxKernArgAlign: 1 isEntryFunction: true diff --git a/llvm/test/CodeGen/AMDGPU/greedy-alloc-fail-sgpr1024-spill.mir b/llvm/test/CodeGen/AMDGPU/greedy-alloc-fail-sgpr1024-spill.mir index bdd89a907790..dde84af57ed2 100644 --- a/llvm/test/CodeGen/AMDGPU/greedy-alloc-fail-sgpr1024-spill.mir +++ b/llvm/test/CodeGen/AMDGPU/greedy-alloc-fail-sgpr1024-spill.mir @@ -13,6 +13,7 @@ name: greedy_fail_alloc_sgpr1024_spill tracksRegLiveness: true frameInfo: + adjustsStack: true hasCalls: true machineFunctionInfo: explicitKernArgSize: 16 diff --git a/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir b/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir index 2ccc24152a9f..fdfc9b043cc9 100644 --- a/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir +++ b/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir @@ -24,6 +24,7 @@ registers: - { id: 10, class: sreg_64_xexec, preferred-register: '$vcc' } frameInfo: maxAlignment: 1 + adjustsStack: true hasCalls: true machineFunctionInfo: maxKernArgAlign: 1 diff --git a/llvm/test/CodeGen/AMDGPU/sched-crash-dbg-value.mir b/llvm/test/CodeGen/AMDGPU/sched-crash-dbg-value.mir index c0d199920bd9..09037709d51d 100644 --- a/llvm/test/CodeGen/AMDGPU/sched-crash-dbg-value.mir +++ b/llvm/test/CodeGen/AMDGPU/sched-crash-dbg-value.mir @@ -181,6 +181,8 @@ legalized: false regBankSelected: false selected: false tracksRegLiveness: true +frameInfo: + adjustsStack: true liveins: - { reg: '$vgpr0', virtual-reg: '%0' } - { reg: '$vgpr1', virtual-reg: '%1' } diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-spill-wrong-stack-id.mir b/llvm/test/CodeGen/AMDGPU/sgpr-spill-wrong-stack-id.mir index efbdbca9da6b..c6ccbd99bf89 100644 --- a/llvm/test/CodeGen/AMDGPU/sgpr-spill-wrong-stack-id.mir +++ b/llvm/test/CodeGen/AMDGPU/sgpr-spill-wrong-stack-id.mir @@ -78,6 +78,7 @@ name: sgpr_spill_wrong_stack_id tracksRegLiveness: true frameInfo: + adjustsStack: true hasCalls: true machineFunctionInfo: scratchRSrcReg: $sgpr0_sgpr1_sgpr2_sgpr3 diff --git a/llvm/test/CodeGen/AMDGPU/snippet-copy-bundle-regression.mir b/llvm/test/CodeGen/AMDGPU/snippet-copy-bundle-regression.mir index 355829825146..f8ec6bb5d943 100644 --- a/llvm/test/CodeGen/AMDGPU/snippet-copy-bundle-regression.mir +++ b/llvm/test/CodeGen/AMDGPU/snippet-copy-bundle-regression.mir @@ -21,6 +21,7 @@ name: kernel tracksRegLiveness: true frameInfo: + adjustsStack: true hasCalls: true machineFunctionInfo: isEntryFunction: true diff --git a/llvm/test/CodeGen/AMDGPU/virtregrewrite-undef-identity-copy.mir b/llvm/test/CodeGen/AMDGPU/virtregrewrite-undef-identity-copy.mir index 3d9db687ffa1..6659e9532376 100644 --- a/llvm/test/CodeGen/AMDGPU/virtregrewrite-undef-identity-copy.mir +++ b/llvm/test/CodeGen/AMDGPU/virtregrewrite-undef-identity-copy.mir @@ -20,6 +20,7 @@ name: undef_identity_copy tracksRegLiveness: true frameInfo: maxAlignment: 4 + adjustsStack: true hasCalls: true machineFunctionInfo: isEntryFunction: true diff --git a/llvm/test/CodeGen/ARM/no-register-coalescing-in-returnsTwice.mir b/llvm/test/CodeGen/ARM/no-register-coalescing-in-returnsTwice.mir index 5c59566247d8..b4bbb9be8ae4 100644 --- a/llvm/test/CodeGen/ARM/no-register-coalescing-in-returnsTwice.mir +++ b/llvm/test/CodeGen/ARM/no-register-coalescing-in-returnsTwice.mir @@ -86,6 +86,8 @@ --- name: main exposesReturnsTwice: true +frameInfo: + adjustsStack: true stack: - { id: 0, name: P0, size: 80, alignment: 8, local-offset: -80 } - { id: 1, name: jb1, size: 160, alignment: 8, local-offset: -240 } diff --git a/llvm/test/CodeGen/Hexagon/regalloc-bad-undef.mir b/llvm/test/CodeGen/Hexagon/regalloc-bad-undef.mir index 67f4dd72ea0b..9468b18bf8e4 100644 --- a/llvm/test/CodeGen/Hexagon/regalloc-bad-undef.mir +++ b/llvm/test/CodeGen/Hexagon/regalloc-bad-undef.mir @@ -135,7 +135,7 @@ frameInfo: stackSize: 0 offsetAdjustment: 0 maxAlignment: 0 - adjustsStack: false + adjustsStack: true hasCalls: true maxCallFrameSize: 0 hasOpaqueSPAdjustment: false diff --git a/llvm/test/CodeGen/SystemZ/RAbasic-invalid-LR-update.mir b/llvm/test/CodeGen/SystemZ/RAbasic-invalid-LR-update.mir index 3b308ce3d0d2..adeec15b1755 100644 --- a/llvm/test/CodeGen/SystemZ/RAbasic-invalid-LR-update.mir +++ b/llvm/test/CodeGen/SystemZ/RAbasic-invalid-LR-update.mir @@ -25,6 +25,8 @@ name: autogen_SD21418 alignment: 4 tracksRegLiveness: true +frameInfo: + adjustsStack: true registers: - { id: 0, class: vr128bit } - { id: 1, class: vr128bit } diff --git a/llvm/test/CodeGen/SystemZ/clear-liverange-spillreg.mir b/llvm/test/CodeGen/SystemZ/clear-liverange-spillreg.mir index 7ff7d9b8b709..197c3d8551fc 100644 --- a/llvm/test/CodeGen/SystemZ/clear-liverange-spillreg.mir +++ b/llvm/test/CodeGen/SystemZ/clear-liverange-spillreg.mir @@ -157,6 +157,7 @@ registers: - { id: 129, class: grx32bit } - { id: 130, class: fp64bit } frameInfo: + adjustsStack: true hasCalls: true body: | bb.0: diff --git a/llvm/test/CodeGen/SystemZ/int-cmp-56.mir b/llvm/test/CodeGen/SystemZ/int-cmp-56.mir index e52fd44ae47d..3e00b6065eb9 100644 --- a/llvm/test/CodeGen/SystemZ/int-cmp-56.mir +++ b/llvm/test/CodeGen/SystemZ/int-cmp-56.mir @@ -48,6 +48,7 @@ liveins: - { reg: '$r2d', virtual-reg: '%0' } frameInfo: maxAlignment: 1 + adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | @@ -125,6 +126,7 @@ liveins: - { reg: '$r2d', virtual-reg: '%0' } frameInfo: maxAlignment: 1 + adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | @@ -202,6 +204,7 @@ liveins: - { reg: '$r2d', virtual-reg: '%0' } frameInfo: maxAlignment: 1 + adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | @@ -279,6 +282,7 @@ liveins: - { reg: '$r2d', virtual-reg: '%0' } frameInfo: maxAlignment: 1 + adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | diff --git a/llvm/test/CodeGen/SystemZ/regcoal-subranges-update.mir b/llvm/test/CodeGen/SystemZ/regcoal-subranges-update.mir index f709b70ff1b7..bf5855010bf9 100644 --- a/llvm/test/CodeGen/SystemZ/regcoal-subranges-update.mir +++ b/llvm/test/CodeGen/SystemZ/regcoal-subranges-update.mir @@ -49,6 +49,8 @@ body: | --- name: segfault tracksRegLiveness: true +frameInfo: + adjustsStack: true liveins: [] body: | ; CHECK-LABEL: name: segfault diff --git a/llvm/test/CodeGen/X86/callbr-asm-kill.mir b/llvm/test/CodeGen/X86/callbr-asm-kill.mir index 86c58c4715ed..0dded37c97af 100644 --- a/llvm/test/CodeGen/X86/callbr-asm-kill.mir +++ b/llvm/test/CodeGen/X86/callbr-asm-kill.mir @@ -45,6 +45,7 @@ liveins: - { reg: '$rsi', virtual-reg: '%3' } frameInfo: maxAlignment: 1 + adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | diff --git a/llvm/test/CodeGen/X86/late-remat-update.mir b/llvm/test/CodeGen/X86/late-remat-update.mir index 84a78f84728c..dd4e99c6df14 100644 --- a/llvm/test/CodeGen/X86/late-remat-update.mir +++ b/llvm/test/CodeGen/X86/late-remat-update.mir @@ -66,6 +66,7 @@ registers: liveins: - { reg: '$edi', virtual-reg: '%0' } frameInfo: + adjustsStack: true hasCalls: true body: | bb.0.entry: diff --git a/llvm/test/CodeGen/X86/limit-split-cost.mir b/llvm/test/CodeGen/X86/limit-split-cost.mir index 6f5329e5b332..7ec0404e0f73 100644 --- a/llvm/test/CodeGen/X86/limit-split-cost.mir +++ b/llvm/test/CodeGen/X86/limit-split-cost.mir @@ -86,6 +86,7 @@ registers: liveins: - { reg: '$edi', virtual-reg: '%0' } frameInfo: + adjustsStack: true hasCalls: true body: | bb.0.entry: diff --git a/llvm/test/CodeGen/X86/regalloc-copy-hints.mir b/llvm/test/CodeGen/X86/regalloc-copy-hints.mir index 13b5a541fa22..d09bcd6a6b40 100644 --- a/llvm/test/CodeGen/X86/regalloc-copy-hints.mir +++ b/llvm/test/CodeGen/X86/regalloc-copy-hints.mir @@ -103,6 +103,7 @@ registers: - { id: 82, class: gr32 } frameInfo: maxAlignment: 4 + adjustsStack: true hasCalls: true fixedStack: - { id: 0, size: 4, alignment: 4, stack-id: default, isImmutable: true } diff --git a/llvm/test/CodeGen/X86/statepoint-fastregalloc.mir b/llvm/test/CodeGen/X86/statepoint-fastregalloc.mir index 02c931067300..8bac14018a7d 100644 --- a/llvm/test/CodeGen/X86/statepoint-fastregalloc.mir +++ b/llvm/test/CodeGen/X86/statepoint-fastregalloc.mir @@ -6,6 +6,8 @@ --- name: test_relocate tracksRegLiveness: true +frameInfo: + adjustsStack: true body: | bb.0.entry: liveins: $rdi @@ -25,6 +27,8 @@ body: | --- name: test_relocate_multi_regmasks tracksRegLiveness: true +frameInfo: + adjustsStack: true body: | bb.0.entry: liveins: $rdi diff --git a/llvm/test/CodeGen/X86/statepoint-invoke-ra-enter-at-end.mir b/llvm/test/CodeGen/X86/statepoint-invoke-ra-enter-at-end.mir index 11968f17c70a..5f05270729fd 100644 --- a/llvm/test/CodeGen/X86/statepoint-invoke-ra-enter-at-end.mir +++ b/llvm/test/CodeGen/X86/statepoint-invoke-ra-enter-at-end.mir @@ -231,7 +231,7 @@ frameInfo: stackSize: 0 offsetAdjustment: 0 maxAlignment: 1 - adjustsStack: false + adjustsStack: true hasCalls: true stackProtector: '' maxCallFrameSize: 4294967295 diff --git a/llvm/test/CodeGen/X86/statepoint-invoke-ra-hoist-copies.mir b/llvm/test/CodeGen/X86/statepoint-invoke-ra-hoist-copies.mir index aae2f3870138..cf9128260f19 100644 --- a/llvm/test/CodeGen/X86/statepoint-invoke-ra-hoist-copies.mir +++ b/llvm/test/CodeGen/X86/statepoint-invoke-ra-hoist-copies.mir @@ -398,7 +398,7 @@ frameInfo: stackSize: 0 offsetAdjustment: 0 maxAlignment: 1 - adjustsStack: false + adjustsStack: true hasCalls: true stackProtector: '' maxCallFrameSize: 4294967295 diff --git a/llvm/test/CodeGen/X86/statepoint-invoke-ra-inline-spiller.mir b/llvm/test/CodeGen/X86/statepoint-invoke-ra-inline-spiller.mir index 87f5f0f96c50..fcebc69d9b2e 100644 --- a/llvm/test/CodeGen/X86/statepoint-invoke-ra-inline-spiller.mir +++ b/llvm/test/CodeGen/X86/statepoint-invoke-ra-inline-spiller.mir @@ -175,7 +175,7 @@ frameInfo: stackSize: 0 offsetAdjustment: 0 maxAlignment: 4 - adjustsStack: false + adjustsStack: true hasCalls: true stackProtector: '' maxCallFrameSize: 4294967295 diff --git a/llvm/test/CodeGen/X86/statepoint-invoke-ra-remove-back-copies.mir b/llvm/test/CodeGen/X86/statepoint-invoke-ra-remove-back-copies.mir index 49253968fcca..8bb39a03f7e3 100644 --- a/llvm/test/CodeGen/X86/statepoint-invoke-ra-remove-back-copies.mir +++ b/llvm/test/CodeGen/X86/statepoint-invoke-ra-remove-back-copies.mir @@ -226,7 +226,7 @@ frameInfo: stackSize: 0 offsetAdjustment: 0 maxAlignment: 4 - adjustsStack: false + adjustsStack: true hasCalls: true stackProtector: '' maxCallFrameSize: 4294967295 diff --git a/llvm/test/CodeGen/X86/statepoint-invoke-ra.mir b/llvm/test/CodeGen/X86/statepoint-invoke-ra.mir index 858ff3f1888b..da651039ce21 100644 --- a/llvm/test/CodeGen/X86/statepoint-invoke-ra.mir +++ b/llvm/test/CodeGen/X86/statepoint-invoke-ra.mir @@ -172,7 +172,7 @@ frameInfo: stackSize: 0 offsetAdjustment: 0 maxAlignment: 4 - adjustsStack: false + adjustsStack: true hasCalls: true stackProtector: '' maxCallFrameSize: 4294967295 diff --git a/llvm/test/CodeGen/X86/statepoint-vreg-folding.mir b/llvm/test/CodeGen/X86/statepoint-vreg-folding.mir index e24d5e8af1f5..d40a9a06d162 100644 --- a/llvm/test/CodeGen/X86/statepoint-vreg-folding.mir +++ b/llvm/test/CodeGen/X86/statepoint-vreg-folding.mir @@ -114,7 +114,7 @@ frameInfo: stackSize: 0 offsetAdjustment: 0 maxAlignment: 8 - adjustsStack: false + adjustsStack: true hasCalls: true stackProtector: '' maxCallFrameSize: 4294967295 diff --git a/llvm/test/DebugInfo/MIR/InstrRef/memory-operand-folding-tieddef.mir b/llvm/test/DebugInfo/MIR/InstrRef/memory-operand-folding-tieddef.mir index cece656d0897..5ebd1a89ae92 100644 --- a/llvm/test/DebugInfo/MIR/InstrRef/memory-operand-folding-tieddef.mir +++ b/llvm/test/DebugInfo/MIR/InstrRef/memory-operand-folding-tieddef.mir @@ -100,6 +100,7 @@ registers: - { id: 38, class: gr8 } frameInfo: maxAlignment: 1 + adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | diff --git a/llvm/test/DebugInfo/MIR/InstrRef/memory-operand-load-folding.mir b/llvm/test/DebugInfo/MIR/InstrRef/memory-operand-load-folding.mir index f0af93804048..b0bff30c4c82 100644 --- a/llvm/test/DebugInfo/MIR/InstrRef/memory-operand-load-folding.mir +++ b/llvm/test/DebugInfo/MIR/InstrRef/memory-operand-load-folding.mir @@ -87,6 +87,7 @@ liveins: - { reg: '$edi', virtual-reg: '%0' } - { reg: '$xmm0', virtual-reg: '%1' } frameInfo: + adjustsStack: true hasCalls: true body: | bb.0.if.then: diff --git a/llvm/test/DebugInfo/MIR/InstrRef/phi-coalesce-subreg.mir b/llvm/test/DebugInfo/MIR/InstrRef/phi-coalesce-subreg.mir index 51d3f7e1a6a4..d73d8a375906 100644 --- a/llvm/test/DebugInfo/MIR/InstrRef/phi-coalesce-subreg.mir +++ b/llvm/test/DebugInfo/MIR/InstrRef/phi-coalesce-subreg.mir @@ -97,6 +97,7 @@ liveins: - { reg: '$esi', virtual-reg: '%4' } frameInfo: maxAlignment: 1 + adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | diff --git a/llvm/test/DebugInfo/MIR/InstrRef/phi-coalescing.mir b/llvm/test/DebugInfo/MIR/InstrRef/phi-coalescing.mir index bc1c7ebac6ce..6460263c6025 100644 --- a/llvm/test/DebugInfo/MIR/InstrRef/phi-coalescing.mir +++ b/llvm/test/DebugInfo/MIR/InstrRef/phi-coalescing.mir @@ -106,6 +106,7 @@ liveins: - { reg: '$rsi', virtual-reg: '%5' } frameInfo: maxAlignment: 1 + adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | diff --git a/llvm/test/DebugInfo/MIR/InstrRef/phi-on-stack-coalesced.mir b/llvm/test/DebugInfo/MIR/InstrRef/phi-on-stack-coalesced.mir index d59333e73fbc..68c9bf6c89dd 100644 --- a/llvm/test/DebugInfo/MIR/InstrRef/phi-on-stack-coalesced.mir +++ b/llvm/test/DebugInfo/MIR/InstrRef/phi-on-stack-coalesced.mir @@ -70,6 +70,7 @@ liveins: - { reg: '$esi', virtual-reg: '%2' } frameInfo: maxAlignment: 1 + adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | diff --git a/llvm/test/DebugInfo/MIR/InstrRef/phi-on-stack-coalesced2.mir b/llvm/test/DebugInfo/MIR/InstrRef/phi-on-stack-coalesced2.mir index ab2647d3b45a..cf17af4ba430 100644 --- a/llvm/test/DebugInfo/MIR/InstrRef/phi-on-stack-coalesced2.mir +++ b/llvm/test/DebugInfo/MIR/InstrRef/phi-on-stack-coalesced2.mir @@ -71,6 +71,7 @@ liveins: - { reg: '$esi', virtual-reg: '%2' } frameInfo: maxAlignment: 1 + adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | diff --git a/llvm/test/DebugInfo/MIR/InstrRef/phi-regallocd-to-stack.mir b/llvm/test/DebugInfo/MIR/InstrRef/phi-regallocd-to-stack.mir index 0fe80980e4e6..cb35bd892eea 100644 --- a/llvm/test/DebugInfo/MIR/InstrRef/phi-regallocd-to-stack.mir +++ b/llvm/test/DebugInfo/MIR/InstrRef/phi-regallocd-to-stack.mir @@ -65,6 +65,7 @@ liveins: - { reg: '$esi', virtual-reg: '%2' } frameInfo: maxAlignment: 1 + adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | diff --git a/llvm/test/DebugInfo/MIR/InstrRef/phi-through-regalloc.mir b/llvm/test/DebugInfo/MIR/InstrRef/phi-through-regalloc.mir index 2a031b295a1e..61dcec49b74c 100644 --- a/llvm/test/DebugInfo/MIR/InstrRef/phi-through-regalloc.mir +++ b/llvm/test/DebugInfo/MIR/InstrRef/phi-through-regalloc.mir @@ -94,6 +94,7 @@ liveins: - { reg: '$esi', virtual-reg: '%2' } frameInfo: maxAlignment: 1 + adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | diff --git a/llvm/test/DebugInfo/MIR/InstrRef/stack-coloring-dbg-phi.mir b/llvm/test/DebugInfo/MIR/InstrRef/stack-coloring-dbg-phi.mir index 47a7b460e43e..e80ed2e3e8eb 100644 --- a/llvm/test/DebugInfo/MIR/InstrRef/stack-coloring-dbg-phi.mir +++ b/llvm/test/DebugInfo/MIR/InstrRef/stack-coloring-dbg-phi.mir @@ -106,6 +106,7 @@ liveins: - { reg: '$rdi', virtual-reg: '%15' } frameInfo: maxAlignment: 8 + adjustsStack: true hasCalls: true stack: - { id: 0, size: 8, alignment: 8 } diff --git a/llvm/test/DebugInfo/MIR/InstrRef/survives-livedebugvars.mir b/llvm/test/DebugInfo/MIR/InstrRef/survives-livedebugvars.mir index 3e806e43ca9e..6dbd2cd4faa7 100644 --- a/llvm/test/DebugInfo/MIR/InstrRef/survives-livedebugvars.mir +++ b/llvm/test/DebugInfo/MIR/InstrRef/survives-livedebugvars.mir @@ -113,6 +113,7 @@ liveins: - { reg: '$rdi', virtual-reg: '%2' } - { reg: '$esi', virtual-reg: '%4' } frameInfo: + adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | diff --git a/llvm/test/DebugInfo/MIR/Mips/livedebugvars-stop-trimming-loc.mir b/llvm/test/DebugInfo/MIR/Mips/livedebugvars-stop-trimming-loc.mir index 35ab906efc90..5df70096e930 100644 --- a/llvm/test/DebugInfo/MIR/Mips/livedebugvars-stop-trimming-loc.mir +++ b/llvm/test/DebugInfo/MIR/Mips/livedebugvars-stop-trimming-loc.mir @@ -72,6 +72,8 @@ name: fn2 alignment: 4 tracksRegLiveness: true +frameInfo: + adjustsStack: true registers: - { id: 0, class: gpr32, preferred-register: '' } - { id: 1, class: gpr32, preferred-register: '' } diff --git a/llvm/test/DebugInfo/MIR/X86/live-debug-vars-unused-arg-debugonly.mir b/llvm/test/DebugInfo/MIR/X86/live-debug-vars-unused-arg-debugonly.mir index 3cb9da8fdfe3..4d48774a78dd 100644 --- a/llvm/test/DebugInfo/MIR/X86/live-debug-vars-unused-arg-debugonly.mir +++ b/llvm/test/DebugInfo/MIR/X86/live-debug-vars-unused-arg-debugonly.mir @@ -116,7 +116,7 @@ frameInfo: stackSize: 0 offsetAdjustment: 0 maxAlignment: 0 - adjustsStack: false + adjustsStack: true hasCalls: true stackProtector: '' maxCallFrameSize: 4294967295 diff --git a/llvm/test/DebugInfo/MIR/X86/live-debug-vars-unused-arg.mir b/llvm/test/DebugInfo/MIR/X86/live-debug-vars-unused-arg.mir index 35d12b52af89..e618f48f527b 100644 --- a/llvm/test/DebugInfo/MIR/X86/live-debug-vars-unused-arg.mir +++ b/llvm/test/DebugInfo/MIR/X86/live-debug-vars-unused-arg.mir @@ -114,7 +114,7 @@ frameInfo: stackSize: 0 offsetAdjustment: 0 maxAlignment: 0 - adjustsStack: false + adjustsStack: true hasCalls: true stackProtector: '' maxCallFrameSize: 4294967295 diff --git a/llvm/test/DebugInfo/MIR/X86/livedebugvars-crossbb-interval.mir b/llvm/test/DebugInfo/MIR/X86/livedebugvars-crossbb-interval.mir index 037306a2ca95..42ee73dde965 100644 --- a/llvm/test/DebugInfo/MIR/X86/livedebugvars-crossbb-interval.mir +++ b/llvm/test/DebugInfo/MIR/X86/livedebugvars-crossbb-interval.mir @@ -100,6 +100,7 @@ liveins: - { reg: '$rdi', virtual-reg: '%2' } - { reg: '$esi', virtual-reg: '%4' } frameInfo: + adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | diff --git a/llvm/test/DebugInfo/X86/live-debug-vars-intervals.mir b/llvm/test/DebugInfo/X86/live-debug-vars-intervals.mir index c5b6d7381ace..3beaf8996e4f 100644 --- a/llvm/test/DebugInfo/X86/live-debug-vars-intervals.mir +++ b/llvm/test/DebugInfo/X86/live-debug-vars-intervals.mir @@ -99,6 +99,8 @@ --- name: f1 tracksRegLiveness: true +frameInfo: + adjustsStack: true stack: - { id: 0, name: x.addr, type: default, offset: 0, size: 4, alignment: 4, stack-id: default, callee-saved-register: '', callee-saved-restored: true, @@ -127,6 +129,8 @@ body: | --- name: f2 tracksRegLiveness: true +frameInfo: + adjustsStack: true stack: - { id: 0, name: x.addr, type: default, offset: 0, size: 4, alignment: 4, stack-id: default, callee-saved-register: '', callee-saved-restored: true, diff --git a/llvm/test/MachineVerifier/test_adjustsstack.mir b/llvm/test/MachineVerifier/test_adjustsstack.mir new file mode 100644 index 000000000000..d333737e000c --- /dev/null +++ b/llvm/test/MachineVerifier/test_adjustsstack.mir @@ -0,0 +1,26 @@ +# RUN: not --crash llc -o - -start-before=twoaddressinstruction -verify-machineinstrs %s 2>&1 \ +# RUN: | FileCheck %s +# REQUIRES: aarch64-registered-target +--- | + target triple = "aarch64-unknown-linux" + declare i32 @bar(i32) nounwind + define i32 @foo() nounwind { + call i32 @bar(i32 0) + ret i32 0 + } +... +--- +name: foo +registers: + - { id: 0, class: gpr32 } +body: | + bb.0 (%ir-block.0): + ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp + %0 = COPY $wzr + $w0 = COPY %0 + BL @bar, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit $w0, implicit-def $sp, implicit-def $w0 + ADJCALLSTACKUP 0, 0, implicit-def dead $sp, implicit $sp + $w0 = COPY killed %0 + RET_ReallyLR implicit $w0 +... +# CHECK-LABEL: Bad machine code: AdjustsStack not set in presence of a frame pseudo instruction. -- GitLab From 8cb2d436ca50117026a8dc901c8039d9bd39b507 Mon Sep 17 00:00:00 2001 From: paperchalice Date: Fri, 22 Mar 2024 08:48:22 +0800 Subject: [PATCH 015/306] [Passes] Expose parseSinglePassOption (#86117) BPF and some machine function passes need it. --- llvm/include/llvm/Passes/PassBuilder.h | 7 ++++ llvm/lib/Passes/PassBuilder.cpp | 55 +++++++++++++++----------- 2 files changed, 38 insertions(+), 24 deletions(-) diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h index 8817a2585646..d1232124d5d8 100644 --- a/llvm/include/llvm/Passes/PassBuilder.h +++ b/llvm/include/llvm/Passes/PassBuilder.h @@ -672,6 +672,13 @@ public: return Result; } + /// Handle passes only accept one bool-valued parameter. + /// + /// \return false when Params is empty. + static Expected parseSinglePassOption(StringRef Params, + StringRef OptionName, + StringRef PassName); + private: // O1 pass pipeline FunctionPassManager diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index 9d98ae7dde52..c5465da8b4a5 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -600,8 +600,9 @@ Expected parseLoopUnrollOptions(StringRef Params) { return UnrollOpts; } -Expected parseSinglePassOption(StringRef Params, StringRef OptionName, - StringRef PassName) { +Expected PassBuilder::parseSinglePassOption(StringRef Params, + StringRef OptionName, + StringRef PassName) { bool Result = false; while (!Params.empty()) { StringRef ParamName; @@ -620,24 +621,28 @@ Expected parseSinglePassOption(StringRef Params, StringRef OptionName, } Expected parseGlobalDCEPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "vfe-linkage-unit-visibility", "GlobalDCE"); + return PassBuilder::parseSinglePassOption( + Params, "vfe-linkage-unit-visibility", "GlobalDCE"); } Expected parseCGProfilePassOptions(StringRef Params) { - return parseSinglePassOption(Params, "in-lto-post-link", "CGProfile"); + return PassBuilder::parseSinglePassOption(Params, "in-lto-post-link", + "CGProfile"); } Expected parseInlinerPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "only-mandatory", "InlinerPass"); + return PassBuilder::parseSinglePassOption(Params, "only-mandatory", + "InlinerPass"); } Expected parseCoroSplitPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "reuse-storage", "CoroSplitPass"); + return PassBuilder::parseSinglePassOption(Params, "reuse-storage", + "CoroSplitPass"); } Expected parsePostOrderFunctionAttrsPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "skip-non-recursive-function-attrs", - "PostOrderFunctionAttrs"); + return PassBuilder::parseSinglePassOption( + Params, "skip-non-recursive-function-attrs", "PostOrderFunctionAttrs"); } Expected parseCFGuardPassOptions(StringRef Params) { @@ -661,19 +666,21 @@ Expected parseCFGuardPassOptions(StringRef Params) { } Expected parseEarlyCSEPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "memssa", "EarlyCSE"); + return PassBuilder::parseSinglePassOption(Params, "memssa", "EarlyCSE"); } Expected parseEntryExitInstrumenterPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "post-inline", "EntryExitInstrumenter"); + return PassBuilder::parseSinglePassOption(Params, "post-inline", + "EntryExitInstrumenter"); } Expected parseLoopExtractorPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "single", "LoopExtractor"); + return PassBuilder::parseSinglePassOption(Params, "single", "LoopExtractor"); } Expected parseLowerMatrixIntrinsicsPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "minimal", "LowerMatrixIntrinsics"); + return PassBuilder::parseSinglePassOption(Params, "minimal", + "LowerMatrixIntrinsics"); } Expected parseASanPassOptions(StringRef Params) { @@ -1013,13 +1020,13 @@ parseStackLifetimeOptions(StringRef Params) { } Expected parseDependenceAnalysisPrinterOptions(StringRef Params) { - return parseSinglePassOption(Params, "normalized-results", - "DependenceAnalysisPrinter"); + return PassBuilder::parseSinglePassOption(Params, "normalized-results", + "DependenceAnalysisPrinter"); } Expected parseSeparateConstOffsetFromGEPPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "lower-gep", - "SeparateConstOffsetFromGEP"); + return PassBuilder::parseSinglePassOption(Params, "lower-gep", + "SeparateConstOffsetFromGEP"); } Expected @@ -1035,13 +1042,13 @@ parseFunctionSimplificationPipelineOptions(StringRef Params) { } Expected parseMemorySSAPrinterPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "no-ensure-optimized-uses", - "MemorySSAPrinterPass"); + return PassBuilder::parseSinglePassOption(Params, "no-ensure-optimized-uses", + "MemorySSAPrinterPass"); } Expected parseSpeculativeExecutionPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "only-if-divergent-target", - "SpeculativeExecutionPass"); + return PassBuilder::parseSinglePassOption(Params, "only-if-divergent-target", + "SpeculativeExecutionPass"); } Expected parseMemProfUsePassOptions(StringRef Params) { @@ -1062,13 +1069,13 @@ Expected parseMemProfUsePassOptions(StringRef Params) { } Expected parseStructuralHashPrinterPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "detailed", - "StructuralHashPrinterPass"); + return PassBuilder::parseSinglePassOption(Params, "detailed", + "StructuralHashPrinterPass"); } Expected parseWinEHPrepareOptions(StringRef Params) { - return parseSinglePassOption(Params, "demote-catchswitch-only", - "WinEHPreparePass"); + return PassBuilder::parseSinglePassOption(Params, "demote-catchswitch-only", + "WinEHPreparePass"); } Expected parseGlobalMergeOptions(StringRef Params) { -- GitLab From a2dfc9ac7da23ccf0077081c8825a23aed1df0c0 Mon Sep 17 00:00:00 2001 From: paperchalice Date: Fri, 22 Mar 2024 08:49:29 +0800 Subject: [PATCH 016/306] [NewPM][AMDGPU] Add AMDGPUPassRegistry.def (#86095) Move the pass registry to a separate file, prepare for porting dag-isel. --- llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def | 73 +++++++++++ .../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 115 +++--------------- .../AMDGPU/global_atomic_optimizer_fp_rtn.ll | 4 +- .../global_atomics_iterative_scan_fp.ll | 4 +- .../global_atomics_optimizer_fp_no_rtn.ll | 4 +- 5 files changed, 96 insertions(+), 104 deletions(-) create mode 100644 llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def new file mode 100644 index 000000000000..90f36fadf359 --- /dev/null +++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def @@ -0,0 +1,73 @@ +//===- AMDGPUPassRegistry.def - Registry of AMDGPU passes -------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is used as the registry of passes that are part of the +// AMDGPU backend. +// +//===----------------------------------------------------------------------===// + +// NOTE: NO INCLUDE GUARD DESIRED! + +#ifndef MODULE_PASS +#define MODULE_PASS(NAME, CREATE_PASS) +#endif +MODULE_PASS("amdgpu-always-inline", AMDGPUAlwaysInlinePass()) +MODULE_PASS("amdgpu-attributor", AMDGPUAttributorPass(*this)) +MODULE_PASS("amdgpu-lower-buffer-fat-pointers", + AMDGPULowerBufferFatPointersPass(*this)) +MODULE_PASS("amdgpu-lower-ctor-dtor", AMDGPUCtorDtorLoweringPass()) +MODULE_PASS("amdgpu-lower-module-lds", AMDGPULowerModuleLDSPass(*this)) +MODULE_PASS("amdgpu-printf-runtime-binding", AMDGPUPrintfRuntimeBindingPass()) +MODULE_PASS("amdgpu-unify-metadata", AMDGPUUnifyMetadataPass()) +#undef MODULE_PASS + +#ifndef FUNCTION_PASS +#define FUNCTION_PASS(NAME, CREATE_PASS) +#endif +FUNCTION_PASS("amdgpu-codegenprepare", AMDGPUCodeGenPreparePass(*this)) +FUNCTION_PASS("amdgpu-image-intrinsic-opt", + AMDGPUImageIntrinsicOptimizerPass(*this)) +FUNCTION_PASS("amdgpu-lower-kernel-arguments", + AMDGPULowerKernelArgumentsPass(*this)) +FUNCTION_PASS("amdgpu-lower-kernel-attributes", + AMDGPULowerKernelAttributesPass()) +FUNCTION_PASS("amdgpu-simplifylib", AMDGPUSimplifyLibCallsPass()) +FUNCTION_PASS("amdgpu-promote-alloca", AMDGPUPromoteAllocaPass(*this)) +FUNCTION_PASS("amdgpu-promote-alloca-to-vector", + AMDGPUPromoteAllocaToVectorPass(*this)) +FUNCTION_PASS("amdgpu-promote-kernel-arguments", + AMDGPUPromoteKernelArgumentsPass()) +FUNCTION_PASS("amdgpu-rewrite-undef-for-phi", AMDGPURewriteUndefForPHIPass()) +FUNCTION_PASS("amdgpu-unify-divergent-exit-nodes", + AMDGPUUnifyDivergentExitNodesPass()) +FUNCTION_PASS("amdgpu-usenative", AMDGPUUseNativeCallsPass()) +#undef FUNCTION_PASS + +#ifndef FUNCTION_ANALYSIS +#define FUNCTION_ANALYSIS(NAME, CREATE_PASS) +#endif + +#ifndef FUNCTION_ALIAS_ANALYSIS +#define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS) \ + FUNCTION_ANALYSIS(NAME, CREATE_PASS) +#endif +FUNCTION_ALIAS_ANALYSIS("amdgpu-aa", AMDGPUAA()) +#undef FUNCTION_ALIAS_ANALYSIS +#undef FUNCTION_ANALYSIS + +#ifndef FUNCTION_PASS_WITH_PARAMS +#define FUNCTION_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS) +#endif +FUNCTION_PASS_WITH_PARAMS( + "amdgpu-atomic-optimizer", + "AMDGPUAtomicOptimizerPass", + [=](ScanOptions Strategy) { + return AMDGPUAtomicOptimizerPass(*this, Strategy); + }, + parseAMDGPUAtomicOptimizerStrategy, "strategy=dpp|iterative|none") +#undef FUNCTION_PASS_WITH_PARAMS diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp index c96625092a76..f7e552177d6f 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp @@ -631,107 +631,26 @@ void AMDGPUTargetMachine::registerDefaultAliasAnalyses(AAManager &AAM) { AAM.registerFunctionAnalysis(); } +static Expected +parseAMDGPUAtomicOptimizerStrategy(StringRef Params) { + if (Params.empty()) + return ScanOptions::Iterative; + Params.consume_front("strategy="); + auto Result = StringSwitch>(Params) + .Case("dpp", ScanOptions::DPP) + .Cases("iterative", "", ScanOptions::Iterative) + .Case("none", ScanOptions::None) + .Default(std::nullopt); + if (Result) + return *Result; + return make_error("invalid parameter", inconvertibleErrorCode()); +} + void AMDGPUTargetMachine::registerPassBuilderCallbacks( PassBuilder &PB, bool PopulateClassToPassNames) { - PB.registerPipelineParsingCallback( - [this](StringRef PassName, ModulePassManager &PM, - ArrayRef) { - if (PassName == "amdgpu-attributor") { - PM.addPass(AMDGPUAttributorPass(*this)); - return true; - } - if (PassName == "amdgpu-unify-metadata") { - PM.addPass(AMDGPUUnifyMetadataPass()); - return true; - } - if (PassName == "amdgpu-printf-runtime-binding") { - PM.addPass(AMDGPUPrintfRuntimeBindingPass()); - return true; - } - if (PassName == "amdgpu-always-inline") { - PM.addPass(AMDGPUAlwaysInlinePass()); - return true; - } - if (PassName == "amdgpu-lower-module-lds") { - PM.addPass(AMDGPULowerModuleLDSPass(*this)); - return true; - } - if (PassName == "amdgpu-lower-buffer-fat-pointers") { - PM.addPass(AMDGPULowerBufferFatPointersPass(*this)); - return true; - } - if (PassName == "amdgpu-lower-ctor-dtor") { - PM.addPass(AMDGPUCtorDtorLoweringPass()); - return true; - } - return false; - }); - PB.registerPipelineParsingCallback( - [this](StringRef PassName, FunctionPassManager &PM, - ArrayRef) { - if (PassName == "amdgpu-simplifylib") { - PM.addPass(AMDGPUSimplifyLibCallsPass()); - return true; - } - if (PassName == "amdgpu-image-intrinsic-opt") { - PM.addPass(AMDGPUImageIntrinsicOptimizerPass(*this)); - return true; - } - if (PassName == "amdgpu-usenative") { - PM.addPass(AMDGPUUseNativeCallsPass()); - return true; - } - if (PassName == "amdgpu-promote-alloca") { - PM.addPass(AMDGPUPromoteAllocaPass(*this)); - return true; - } - if (PassName == "amdgpu-promote-alloca-to-vector") { - PM.addPass(AMDGPUPromoteAllocaToVectorPass(*this)); - return true; - } - if (PassName == "amdgpu-lower-kernel-attributes") { - PM.addPass(AMDGPULowerKernelAttributesPass()); - return true; - } - if (PassName == "amdgpu-promote-kernel-arguments") { - PM.addPass(AMDGPUPromoteKernelArgumentsPass()); - return true; - } - if (PassName == "amdgpu-unify-divergent-exit-nodes") { - PM.addPass(AMDGPUUnifyDivergentExitNodesPass()); - return true; - } - if (PassName == "amdgpu-atomic-optimizer") { - PM.addPass( - AMDGPUAtomicOptimizerPass(*this, AMDGPUAtomicOptimizerStrategy)); - return true; - } - if (PassName == "amdgpu-codegenprepare") { - PM.addPass(AMDGPUCodeGenPreparePass(*this)); - return true; - } - if (PassName == "amdgpu-lower-kernel-arguments") { - PM.addPass(AMDGPULowerKernelArgumentsPass(*this)); - return true; - } - if (PassName == "amdgpu-rewrite-undef-for-phi") { - PM.addPass(AMDGPURewriteUndefForPHIPass()); - return true; - } - return false; - }); - - PB.registerAnalysisRegistrationCallback([](FunctionAnalysisManager &FAM) { - FAM.registerPass([&] { return AMDGPUAA(); }); - }); - PB.registerParseAACallback([](StringRef AAName, AAManager &AAM) { - if (AAName == "amdgpu-aa") { - AAM.registerFunctionAnalysis(); - return true; - } - return false; - }); +#define GET_PASS_REGISTRY "AMDGPUPassRegistry.def" +#include "llvm/Passes/TargetPassRegistry.inc" PB.registerPipelineStartEPCallback( [](ModulePassManager &PM, OptimizationLevel Level) { diff --git a/llvm/test/CodeGen/AMDGPU/global_atomic_optimizer_fp_rtn.ll b/llvm/test/CodeGen/AMDGPU/global_atomic_optimizer_fp_rtn.ll index 538ef42121b8..b7a91f6fa96d 100644 --- a/llvm/test/CodeGen/AMDGPU/global_atomic_optimizer_fp_rtn.ll +++ b/llvm/test/CodeGen/AMDGPU/global_atomic_optimizer_fp_rtn.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -mtriple=amdgcn-- -mcpu=gfx906 -amdgpu-atomic-optimizer-strategy=Iterative -passes='amdgpu-atomic-optimizer,verify' %s | FileCheck --check-prefixes=IR,IR-ITERATIVE %s -; RUN: opt -S -mtriple=amdgcn-- -mcpu=gfx906 -amdgpu-atomic-optimizer-strategy=DPP -passes='amdgpu-atomic-optimizer,verify' %s | FileCheck --check-prefixes=IR,IR-DPP %s +; RUN: opt -S -mtriple=amdgcn-- -mcpu=gfx906 -passes='amdgpu-atomic-optimizer,verify' %s | FileCheck --check-prefixes=IR,IR-ITERATIVE %s +; RUN: opt -S -mtriple=amdgcn-- -mcpu=gfx906 -passes='amdgpu-atomic-optimizer,verify' %s | FileCheck --check-prefixes=IR,IR-DPP %s ; Tests various combinations of uniform/divergent address and uniform/divergent value inputs of various types for atomic operations. ; Optimization remains same for Iterative and DPP strategies when value in uniform. These different scan/reduction diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_iterative_scan_fp.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_iterative_scan_fp.ll index fab24e10f810..86e3d9338e07 100644 --- a/llvm/test/CodeGen/AMDGPU/global_atomics_iterative_scan_fp.ll +++ b/llvm/test/CodeGen/AMDGPU/global_atomics_iterative_scan_fp.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -mtriple=amdgcn-- -amdgpu-atomic-optimizer-strategy=Iterative -passes='amdgpu-atomic-optimizer,verify' %s | FileCheck -check-prefix=IR-ITERATIVE %s -; RUN: opt -S -mtriple=amdgcn-- -amdgpu-atomic-optimizer-strategy=DPP -passes='amdgpu-atomic-optimizer,verify' %s | FileCheck -check-prefix=IR-DPP %s +; RUN: opt -S -mtriple=amdgcn-- -passes='amdgpu-atomic-optimizer,verify' %s | FileCheck -check-prefix=IR-ITERATIVE %s +; RUN: opt -S -mtriple=amdgcn-- -passes='amdgpu-atomic-optimizer,verify' %s | FileCheck -check-prefix=IR-DPP %s declare i32 @llvm.amdgcn.workitem.id.x() define amdgpu_kernel void @global_atomic_fadd_uni_value(ptr addrspace(1) %ptr) #0 { ; IR-ITERATIVE-LABEL: @global_atomic_fadd_uni_value( diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_optimizer_fp_no_rtn.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_optimizer_fp_no_rtn.ll index cc7a45cbb6e3..e70d7347890f 100644 --- a/llvm/test/CodeGen/AMDGPU/global_atomics_optimizer_fp_no_rtn.ll +++ b/llvm/test/CodeGen/AMDGPU/global_atomics_optimizer_fp_no_rtn.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -mtriple=amdgcn-- -mcpu=gfx906 -amdgpu-atomic-optimizer-strategy=Iterative -passes='amdgpu-atomic-optimizer,verify' %s | FileCheck --check-prefixes=IR,IR-ITERATIVE %s -; RUN: opt -S -mtriple=amdgcn-- -mcpu=gfx906 -amdgpu-atomic-optimizer-strategy=DPP -passes='amdgpu-atomic-optimizer,verify' %s | FileCheck --check-prefixes=IR,IR-DPP %s +; RUN: opt -S -mtriple=amdgcn-- -mcpu=gfx906 -passes='amdgpu-atomic-optimizer,verify' %s | FileCheck --check-prefixes=IR,IR-ITERATIVE %s +; RUN: opt -S -mtriple=amdgcn-- -mcpu=gfx906 -passes='amdgpu-atomic-optimizer,verify' %s | FileCheck --check-prefixes=IR,IR-DPP %s ; Tests various combinations of uniform/divergent address and uniform/divergent value inputs of various types for atomic operations. ; Optimization remains same for Iterative and DPP strategies when value in uniform. These different scan/reduction -- GitLab From 3e4caa9da4356247444e973eb470a25adae083b0 Mon Sep 17 00:00:00 2001 From: Freddy Ye Date: Fri, 22 Mar 2024 08:52:40 +0800 Subject: [PATCH 017/306] [X86] Support DomainReassignment for APX NDD instructions (#85737) --- llvm/lib/Target/X86/X86DomainReassignment.cpp | 38 ++++++++++++ .../CodeGen/X86/apx/domain-reassignment.mir | 62 +++++++++---------- 2 files changed, 69 insertions(+), 31 deletions(-) diff --git a/llvm/lib/Target/X86/X86DomainReassignment.cpp b/llvm/lib/Target/X86/X86DomainReassignment.cpp index 53c0486c8697..6289b3a1df1f 100644 --- a/llvm/lib/Target/X86/X86DomainReassignment.cpp +++ b/llvm/lib/Target/X86/X86DomainReassignment.cpp @@ -650,6 +650,16 @@ void X86DomainReassignment::initConverters() { createReplacer(X86::AND16rr, X86::KANDWrr); createReplacer(X86::XOR16rr, X86::KXORWrr); + bool HasNDD = STI->hasNDD(); + if (HasNDD) { + createReplacer(X86::SHR16ri_ND, X86::KSHIFTRWri); + createReplacer(X86::SHL16ri_ND, X86::KSHIFTLWri); + createReplacer(X86::NOT16r_ND, X86::KNOTWrr); + createReplacer(X86::OR16rr_ND, X86::KORWrr); + createReplacer(X86::AND16rr_ND, X86::KANDWrr); + createReplacer(X86::XOR16rr_ND, X86::KXORWrr); + } + if (STI->hasBWI()) { createReplacer(X86::MOV32rm, GET_EGPR_IF_ENABLED(X86::KMOVDkm)); createReplacer(X86::MOV64rm, GET_EGPR_IF_ENABLED(X86::KMOVQkm)); @@ -684,6 +694,23 @@ void X86DomainReassignment::initConverters() { createReplacer(X86::XOR32rr, X86::KXORDrr); createReplacer(X86::XOR64rr, X86::KXORQrr); + if (HasNDD) { + createReplacer(X86::SHR32ri_ND, X86::KSHIFTRDri); + createReplacer(X86::SHL32ri_ND, X86::KSHIFTLDri); + createReplacer(X86::ADD32rr_ND, X86::KADDDrr); + createReplacer(X86::NOT32r_ND, X86::KNOTDrr); + createReplacer(X86::OR32rr_ND, X86::KORDrr); + createReplacer(X86::AND32rr_ND, X86::KANDDrr); + createReplacer(X86::XOR32rr_ND, X86::KXORDrr); + createReplacer(X86::SHR64ri_ND, X86::KSHIFTRQri); + createReplacer(X86::SHL64ri_ND, X86::KSHIFTLQri); + createReplacer(X86::ADD64rr_ND, X86::KADDQrr); + createReplacer(X86::NOT64r_ND, X86::KNOTQrr); + createReplacer(X86::OR64rr_ND, X86::KORQrr); + createReplacer(X86::AND64rr_ND, X86::KANDQrr); + createReplacer(X86::XOR64rr_ND, X86::KXORQrr); + } + // TODO: KTEST is not a replacement for TEST due to flag differences. Need // to prove only Z flag is used. // createReplacer(X86::TEST32rr, X86::KTESTDrr); @@ -713,6 +740,17 @@ void X86DomainReassignment::initConverters() { // createReplacer(X86::TEST16rr, X86::KTESTWrr); createReplacer(X86::XOR8rr, X86::KXORBrr); + + if (HasNDD) { + createReplacer(X86::ADD8rr_ND, X86::KADDBrr); + createReplacer(X86::ADD16rr_ND, X86::KADDWrr); + createReplacer(X86::AND8rr_ND, X86::KANDBrr); + createReplacer(X86::NOT8r_ND, X86::KNOTBrr); + createReplacer(X86::OR8rr_ND, X86::KORBrr); + createReplacer(X86::SHR8ri_ND, X86::KSHIFTRBri); + createReplacer(X86::SHL8ri_ND, X86::KSHIFTLBri); + createReplacer(X86::XOR8rr_ND, X86::KXORBrr); + } } #undef GET_EGPR_IF_ENABLED } diff --git a/llvm/test/CodeGen/X86/apx/domain-reassignment.mir b/llvm/test/CodeGen/X86/apx/domain-reassignment.mir index dcd435619990..7352aa2b307f 100644 --- a/llvm/test/CodeGen/X86/apx/domain-reassignment.mir +++ b/llvm/test/CodeGen/X86/apx/domain-reassignment.mir @@ -1,5 +1,5 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py -# RUN: llc -run-pass x86-domain-reassignment -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512dq -o - %s | FileCheck %s +# RUN: llc -run-pass x86-domain-reassignment -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512dq,+ndd -o - %s | FileCheck %s --- | ; ModuleID = '../test/CodeGen/X86/gpr-to-mask.ll' source_filename = "../test/CodeGen/X86/gpr-to-mask.ll" @@ -302,13 +302,13 @@ body: | %6 = COPY %5 %7 = COPY %6.sub_8bit - %12 = SHR8ri %7, 2, implicit-def dead $eflags - %13 = SHL8ri %12, 1, implicit-def dead $eflags - %14 = NOT8r %13 - %15 = OR8rr %14, %12, implicit-def dead $eflags - %16 = AND8rr %15, %13, implicit-def dead $eflags - %17 = XOR8rr %16, %12, implicit-def dead $eflags - %18 = ADD8rr %17, %14, implicit-def dead $eflags + %12 = SHR8ri_ND %7, 2, implicit-def dead $eflags + %13 = SHL8ri_ND %12, 1, implicit-def dead $eflags + %14 = NOT8r_ND %13 + %15 = OR8rr_ND %14, %12, implicit-def dead $eflags + %16 = AND8rr_ND %15, %13, implicit-def dead $eflags + %17 = XOR8rr_ND %16, %12, implicit-def dead $eflags + %18 = ADD8rr_ND %17, %14, implicit-def dead $eflags %8 = IMPLICIT_DEF %9 = INSERT_SUBREG %8, %18, %subreg.sub_8bit_hi @@ -421,12 +421,12 @@ body: | %6 = COPY %5 %7 = COPY %6.sub_16bit - %12 = SHR16ri %7, 2, implicit-def dead $eflags - %13 = SHL16ri %12, 1, implicit-def dead $eflags - %14 = NOT16r %13 - %15 = OR16rr %14, %12, implicit-def dead $eflags - %16 = AND16rr %15, %13, implicit-def dead $eflags - %17 = XOR16rr %16, %12, implicit-def dead $eflags + %12 = SHR16ri_ND %7, 2, implicit-def dead $eflags + %13 = SHL16ri_ND %12, 1, implicit-def dead $eflags + %14 = NOT16r_ND %13 + %15 = OR16rr_ND %14, %12, implicit-def dead $eflags + %16 = AND16rr_ND %15, %13, implicit-def dead $eflags + %17 = XOR16rr_ND %16, %12, implicit-def dead $eflags %8 = IMPLICIT_DEF %9 = INSERT_SUBREG %8, %17, %subreg.sub_16bit @@ -524,14 +524,14 @@ body: | %2 = COPY $zmm1 %5 = MOV32rm %0, 1, $noreg, 0, $noreg - %6 = SHR32ri %5, 2, implicit-def dead $eflags - %7 = SHL32ri %6, 1, implicit-def dead $eflags - %8 = NOT32r %7 - %9 = OR32rr %8, %6, implicit-def dead $eflags - %10 = AND32rr %9, %7, implicit-def dead $eflags - %11 = XOR32rr %10, %6, implicit-def dead $eflags + %6 = SHR32ri_ND %5, 2, implicit-def dead $eflags + %7 = SHL32ri_ND %6, 1, implicit-def dead $eflags + %8 = NOT32r_ND %7 + %9 = OR32rr_ND %8, %6, implicit-def dead $eflags + %10 = AND32rr_ND %9, %7, implicit-def dead $eflags + %11 = XOR32rr_ND %10, %6, implicit-def dead $eflags %12 = ANDN32rr %11, %9, implicit-def dead $eflags - %13 = ADD32rr %12, %11, implicit-def dead $eflags + %13 = ADD32rr_ND %12, %11, implicit-def dead $eflags %3 = COPY %13 %4 = VMOVDQU16Zrrk %2, killed %3, %1 @@ -627,14 +627,14 @@ body: | %2 = COPY $zmm1 %5 = MOV64rm %0, 1, $noreg, 0, $noreg - %6 = SHR64ri %5, 2, implicit-def dead $eflags - %7 = SHL64ri %6, 1, implicit-def dead $eflags - %8 = NOT64r %7 - %9 = OR64rr %8, %6, implicit-def dead $eflags - %10 = AND64rr %9, %7, implicit-def dead $eflags - %11 = XOR64rr %10, %6, implicit-def dead $eflags + %6 = SHR64ri_ND %5, 2, implicit-def dead $eflags + %7 = SHL64ri_ND %6, 1, implicit-def dead $eflags + %8 = NOT64r_ND %7 + %9 = OR64rr_ND %8, %6, implicit-def dead $eflags + %10 = AND64rr_ND %9, %7, implicit-def dead $eflags + %11 = XOR64rr_ND %10, %6, implicit-def dead $eflags %12 = ANDN64rr %11, %9, implicit-def dead $eflags - %13 = ADD64rr %12, %11, implicit-def dead $eflags + %13 = ADD64rr_ND %12, %11, implicit-def dead $eflags %3 = COPY %13 %4 = VMOVDQU8Zrrk %2, killed %3, %1 @@ -712,7 +712,7 @@ body: | %2 = COPY $zmm1 %5 = MOVZX16rm8 %0, 1, $noreg, 0, $noreg - %6 = NOT16r %5 + %6 = NOT16r_ND %5 %3 = COPY %6 %4 = VMOVAPSZrrk %2, killed %3, %1 @@ -785,7 +785,7 @@ body: | %5 = MOVZX32rm8 %0, 1, $noreg, 0, $noreg %6 = MOVZX32rm16 %0, 1, $noreg, 0, $noreg - %7 = ADD32rr %5, %6, implicit-def dead $eflags + %7 = ADD32rr_ND %5, %6, implicit-def dead $eflags %3 = COPY %7 %4 = VMOVDQU16Zrrk %2, killed %3, %1 @@ -858,7 +858,7 @@ body: | %5 = MOVZX64rm8 %0, 1, $noreg, 0, $noreg %6 = MOVZX64rm16 %0, 1, $noreg, 0, $noreg - %7 = ADD64rr %5, %6, implicit-def dead $eflags + %7 = ADD64rr_ND %5, %6, implicit-def dead $eflags %3 = COPY %7 %4 = VMOVDQU8Zrrk %2, killed %3, %1 -- GitLab From 8d7d581ad2a96ebe54aed0e5a626048d2e2a8d2d Mon Sep 17 00:00:00 2001 From: paperchalice Date: Fri, 22 Mar 2024 08:54:11 +0800 Subject: [PATCH 018/306] Revert "[Passes] Expose parseSinglePassOption" (#86225) Reverts llvm/llvm-project#86117 --- llvm/include/llvm/Passes/PassBuilder.h | 7 ---- llvm/lib/Passes/PassBuilder.cpp | 55 +++++++++++--------------- 2 files changed, 24 insertions(+), 38 deletions(-) diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h index d1232124d5d8..8817a2585646 100644 --- a/llvm/include/llvm/Passes/PassBuilder.h +++ b/llvm/include/llvm/Passes/PassBuilder.h @@ -672,13 +672,6 @@ public: return Result; } - /// Handle passes only accept one bool-valued parameter. - /// - /// \return false when Params is empty. - static Expected parseSinglePassOption(StringRef Params, - StringRef OptionName, - StringRef PassName); - private: // O1 pass pipeline FunctionPassManager diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index c5465da8b4a5..9d98ae7dde52 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -600,9 +600,8 @@ Expected parseLoopUnrollOptions(StringRef Params) { return UnrollOpts; } -Expected PassBuilder::parseSinglePassOption(StringRef Params, - StringRef OptionName, - StringRef PassName) { +Expected parseSinglePassOption(StringRef Params, StringRef OptionName, + StringRef PassName) { bool Result = false; while (!Params.empty()) { StringRef ParamName; @@ -621,28 +620,24 @@ Expected PassBuilder::parseSinglePassOption(StringRef Params, } Expected parseGlobalDCEPassOptions(StringRef Params) { - return PassBuilder::parseSinglePassOption( - Params, "vfe-linkage-unit-visibility", "GlobalDCE"); + return parseSinglePassOption(Params, "vfe-linkage-unit-visibility", "GlobalDCE"); } Expected parseCGProfilePassOptions(StringRef Params) { - return PassBuilder::parseSinglePassOption(Params, "in-lto-post-link", - "CGProfile"); + return parseSinglePassOption(Params, "in-lto-post-link", "CGProfile"); } Expected parseInlinerPassOptions(StringRef Params) { - return PassBuilder::parseSinglePassOption(Params, "only-mandatory", - "InlinerPass"); + return parseSinglePassOption(Params, "only-mandatory", "InlinerPass"); } Expected parseCoroSplitPassOptions(StringRef Params) { - return PassBuilder::parseSinglePassOption(Params, "reuse-storage", - "CoroSplitPass"); + return parseSinglePassOption(Params, "reuse-storage", "CoroSplitPass"); } Expected parsePostOrderFunctionAttrsPassOptions(StringRef Params) { - return PassBuilder::parseSinglePassOption( - Params, "skip-non-recursive-function-attrs", "PostOrderFunctionAttrs"); + return parseSinglePassOption(Params, "skip-non-recursive-function-attrs", + "PostOrderFunctionAttrs"); } Expected parseCFGuardPassOptions(StringRef Params) { @@ -666,21 +661,19 @@ Expected parseCFGuardPassOptions(StringRef Params) { } Expected parseEarlyCSEPassOptions(StringRef Params) { - return PassBuilder::parseSinglePassOption(Params, "memssa", "EarlyCSE"); + return parseSinglePassOption(Params, "memssa", "EarlyCSE"); } Expected parseEntryExitInstrumenterPassOptions(StringRef Params) { - return PassBuilder::parseSinglePassOption(Params, "post-inline", - "EntryExitInstrumenter"); + return parseSinglePassOption(Params, "post-inline", "EntryExitInstrumenter"); } Expected parseLoopExtractorPassOptions(StringRef Params) { - return PassBuilder::parseSinglePassOption(Params, "single", "LoopExtractor"); + return parseSinglePassOption(Params, "single", "LoopExtractor"); } Expected parseLowerMatrixIntrinsicsPassOptions(StringRef Params) { - return PassBuilder::parseSinglePassOption(Params, "minimal", - "LowerMatrixIntrinsics"); + return parseSinglePassOption(Params, "minimal", "LowerMatrixIntrinsics"); } Expected parseASanPassOptions(StringRef Params) { @@ -1020,13 +1013,13 @@ parseStackLifetimeOptions(StringRef Params) { } Expected parseDependenceAnalysisPrinterOptions(StringRef Params) { - return PassBuilder::parseSinglePassOption(Params, "normalized-results", - "DependenceAnalysisPrinter"); + return parseSinglePassOption(Params, "normalized-results", + "DependenceAnalysisPrinter"); } Expected parseSeparateConstOffsetFromGEPPassOptions(StringRef Params) { - return PassBuilder::parseSinglePassOption(Params, "lower-gep", - "SeparateConstOffsetFromGEP"); + return parseSinglePassOption(Params, "lower-gep", + "SeparateConstOffsetFromGEP"); } Expected @@ -1042,13 +1035,13 @@ parseFunctionSimplificationPipelineOptions(StringRef Params) { } Expected parseMemorySSAPrinterPassOptions(StringRef Params) { - return PassBuilder::parseSinglePassOption(Params, "no-ensure-optimized-uses", - "MemorySSAPrinterPass"); + return parseSinglePassOption(Params, "no-ensure-optimized-uses", + "MemorySSAPrinterPass"); } Expected parseSpeculativeExecutionPassOptions(StringRef Params) { - return PassBuilder::parseSinglePassOption(Params, "only-if-divergent-target", - "SpeculativeExecutionPass"); + return parseSinglePassOption(Params, "only-if-divergent-target", + "SpeculativeExecutionPass"); } Expected parseMemProfUsePassOptions(StringRef Params) { @@ -1069,13 +1062,13 @@ Expected parseMemProfUsePassOptions(StringRef Params) { } Expected parseStructuralHashPrinterPassOptions(StringRef Params) { - return PassBuilder::parseSinglePassOption(Params, "detailed", - "StructuralHashPrinterPass"); + return parseSinglePassOption(Params, "detailed", + "StructuralHashPrinterPass"); } Expected parseWinEHPrepareOptions(StringRef Params) { - return PassBuilder::parseSinglePassOption(Params, "demote-catchswitch-only", - "WinEHPreparePass"); + return parseSinglePassOption(Params, "demote-catchswitch-only", + "WinEHPreparePass"); } Expected parseGlobalMergeOptions(StringRef Params) { -- GitLab From 718fbbef5f18a2b7e7fc4f842b1452ae9bee581a Mon Sep 17 00:00:00 2001 From: Aiden Grossman Date: Thu, 21 Mar 2024 18:14:18 -0700 Subject: [PATCH 019/306] [llvm-exegesis] Kill process that recieve a signal (#86069) Before this patch, llvm-exegesis would leave processes lingering that experienced signals like segmentation faults. They would up in a signal-delivery-stop state under the ptrace and never exit. This does not cause problems (or at least many) in llvm-exegesis as they are cleaned up after the main process exits, which usually happens quickly. However, in downstream use, when many blocks are being executed (many of which run into signals) within a single process, these processes stay around and can easily exhaust the process limit on some systems. This patch cleans them up by sending SIGKILL after information about the signal that was sent has been gathered. --- .../llvm-exegesis/lib/BenchmarkRunner.cpp | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp index 5c9848f3c688..f0452605eb24 100644 --- a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp +++ b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp @@ -342,7 +342,7 @@ private: return make_error("Failed to attach to the child process: " + Twine(strerror(errno))); - if (wait(NULL) == -1) { + if (waitpid(ParentOrChildPID, NULL, 0) == -1) { return make_error( "Failed to wait for child process to stop after attaching: " + Twine(strerror(errno))); @@ -361,7 +361,7 @@ private: return SendError; int ChildStatus; - if (wait(&ChildStatus) == -1) { + if (waitpid(ParentOrChildPID, &ChildStatus, 0) == -1) { return make_error( "Waiting for the child process to complete failed: " + Twine(strerror(errno))); @@ -401,6 +401,20 @@ private: Twine(strerror(errno))); } + // Send SIGKILL rather than SIGTERM as the child process has no SIGTERM + // handlers to run, and calling SIGTERM would mean that ptrace will force + // it to block in the signal-delivery-stop for the SIGSEGV/other signals, + // and upon exit. + if (kill(ParentOrChildPID, SIGKILL) == -1) + return make_error("Failed to kill child benchmarking proces: " + + Twine(strerror(errno))); + + // Wait for the process to exit so that there are no zombie processes left + // around. + if (waitpid(ParentOrChildPID, NULL, 0) == -1) + return make_error("Failed to wait for process to die: " + + Twine(strerror(errno))); + if (ChildSignalInfo.si_signo == SIGSEGV) return make_error( reinterpret_cast(ChildSignalInfo.si_addr)); -- GitLab From 4d7f28a2c4b187f0bef3877081100786156defc7 Mon Sep 17 00:00:00 2001 From: paperchalice Date: Fri, 22 Mar 2024 10:55:58 +0800 Subject: [PATCH 020/306] [Passes] Expose parseSinglePassOption (#86226) Reland #86225, adjust the name space. --- llvm/include/llvm/Passes/PassBuilder.h | 7 ++ llvm/lib/Passes/PassBuilder.cpp | 89 ++++++++++++++------------ 2 files changed, 55 insertions(+), 41 deletions(-) diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h index 8817a2585646..d1232124d5d8 100644 --- a/llvm/include/llvm/Passes/PassBuilder.h +++ b/llvm/include/llvm/Passes/PassBuilder.h @@ -672,6 +672,13 @@ public: return Result; } + /// Handle passes only accept one bool-valued parameter. + /// + /// \return false when Params is empty. + static Expected parseSinglePassOption(StringRef Params, + StringRef OptionName, + StringRef PassName); + private: // O1 pass pipeline FunctionPassManager diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index 9d98ae7dde52..f60f4eb3f0ef 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -514,6 +514,26 @@ static std::optional parseOptLevel(StringRef S) { .Default(std::nullopt); } +Expected PassBuilder::parseSinglePassOption(StringRef Params, + StringRef OptionName, + StringRef PassName) { + bool Result = false; + while (!Params.empty()) { + StringRef ParamName; + std::tie(ParamName, Params) = Params.split(';'); + + if (ParamName == OptionName) { + Result = true; + } else { + return make_error( + formatv("invalid {1} pass parameter '{0}' ", ParamName, PassName) + .str(), + inconvertibleErrorCode()); + } + } + return Result; +} + namespace { /// Parser of parameters for HardwareLoops pass. @@ -600,44 +620,29 @@ Expected parseLoopUnrollOptions(StringRef Params) { return UnrollOpts; } -Expected parseSinglePassOption(StringRef Params, StringRef OptionName, - StringRef PassName) { - bool Result = false; - while (!Params.empty()) { - StringRef ParamName; - std::tie(ParamName, Params) = Params.split(';'); - - if (ParamName == OptionName) { - Result = true; - } else { - return make_error( - formatv("invalid {1} pass parameter '{0}' ", ParamName, PassName) - .str(), - inconvertibleErrorCode()); - } - } - return Result; -} - Expected parseGlobalDCEPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "vfe-linkage-unit-visibility", "GlobalDCE"); + return PassBuilder::parseSinglePassOption( + Params, "vfe-linkage-unit-visibility", "GlobalDCE"); } Expected parseCGProfilePassOptions(StringRef Params) { - return parseSinglePassOption(Params, "in-lto-post-link", "CGProfile"); + return PassBuilder::parseSinglePassOption(Params, "in-lto-post-link", + "CGProfile"); } Expected parseInlinerPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "only-mandatory", "InlinerPass"); + return PassBuilder::parseSinglePassOption(Params, "only-mandatory", + "InlinerPass"); } Expected parseCoroSplitPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "reuse-storage", "CoroSplitPass"); + return PassBuilder::parseSinglePassOption(Params, "reuse-storage", + "CoroSplitPass"); } Expected parsePostOrderFunctionAttrsPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "skip-non-recursive-function-attrs", - "PostOrderFunctionAttrs"); + return PassBuilder::parseSinglePassOption( + Params, "skip-non-recursive-function-attrs", "PostOrderFunctionAttrs"); } Expected parseCFGuardPassOptions(StringRef Params) { @@ -661,19 +666,21 @@ Expected parseCFGuardPassOptions(StringRef Params) { } Expected parseEarlyCSEPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "memssa", "EarlyCSE"); + return PassBuilder::parseSinglePassOption(Params, "memssa", "EarlyCSE"); } Expected parseEntryExitInstrumenterPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "post-inline", "EntryExitInstrumenter"); + return PassBuilder::parseSinglePassOption(Params, "post-inline", + "EntryExitInstrumenter"); } Expected parseLoopExtractorPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "single", "LoopExtractor"); + return PassBuilder::parseSinglePassOption(Params, "single", "LoopExtractor"); } Expected parseLowerMatrixIntrinsicsPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "minimal", "LowerMatrixIntrinsics"); + return PassBuilder::parseSinglePassOption(Params, "minimal", + "LowerMatrixIntrinsics"); } Expected parseASanPassOptions(StringRef Params) { @@ -1013,13 +1020,13 @@ parseStackLifetimeOptions(StringRef Params) { } Expected parseDependenceAnalysisPrinterOptions(StringRef Params) { - return parseSinglePassOption(Params, "normalized-results", - "DependenceAnalysisPrinter"); + return PassBuilder::parseSinglePassOption(Params, "normalized-results", + "DependenceAnalysisPrinter"); } Expected parseSeparateConstOffsetFromGEPPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "lower-gep", - "SeparateConstOffsetFromGEP"); + return PassBuilder::parseSinglePassOption(Params, "lower-gep", + "SeparateConstOffsetFromGEP"); } Expected @@ -1035,13 +1042,13 @@ parseFunctionSimplificationPipelineOptions(StringRef Params) { } Expected parseMemorySSAPrinterPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "no-ensure-optimized-uses", - "MemorySSAPrinterPass"); + return PassBuilder::parseSinglePassOption(Params, "no-ensure-optimized-uses", + "MemorySSAPrinterPass"); } Expected parseSpeculativeExecutionPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "only-if-divergent-target", - "SpeculativeExecutionPass"); + return PassBuilder::parseSinglePassOption(Params, "only-if-divergent-target", + "SpeculativeExecutionPass"); } Expected parseMemProfUsePassOptions(StringRef Params) { @@ -1062,13 +1069,13 @@ Expected parseMemProfUsePassOptions(StringRef Params) { } Expected parseStructuralHashPrinterPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "detailed", - "StructuralHashPrinterPass"); + return PassBuilder::parseSinglePassOption(Params, "detailed", + "StructuralHashPrinterPass"); } Expected parseWinEHPrepareOptions(StringRef Params) { - return parseSinglePassOption(Params, "demote-catchswitch-only", - "WinEHPreparePass"); + return PassBuilder::parseSinglePassOption(Params, "demote-catchswitch-only", + "WinEHPreparePass"); } Expected parseGlobalMergeOptions(StringRef Params) { -- GitLab From 40beb9b001a3c67c60b98fae9e999dcaa2d88717 Mon Sep 17 00:00:00 2001 From: Kazu Hirata Date: Thu, 21 Mar 2024 20:14:18 -0700 Subject: [PATCH 021/306] [lldb] Handle clang::Language::CIR (#86234) commit e66b670f3bf9312f696e66c31152ae535207d6bb Author: Nathan Lanza Date: Thu Mar 21 19:53:48 2024 -0400 triggers: lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp:478:16: error: enumeration value 'CIR' not handled in switch [-Werror,-Wswitch] This patch teaches lldb to handle clang::Language::CIR the same way as clang::Language::LLVM_IR. --- lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp index 3ac1cf91932c..ebcc3bc99a80 100644 --- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp +++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp @@ -477,6 +477,7 @@ static void ParseLangArgs(LangOptions &Opts, InputKind IK, const char *triple) { // Based on the base language, pick one. switch (IK.getLanguage()) { case clang::Language::Unknown: + case clang::Language::CIR: case clang::Language::LLVM_IR: case clang::Language::RenderScript: llvm_unreachable("Invalid input kind!"); -- GitLab From 4865dab04cad1c5ce47468b0a52ea968e5a5503b Mon Sep 17 00:00:00 2001 From: Kazu Hirata Date: Thu, 21 Mar 2024 20:21:11 -0700 Subject: [PATCH 022/306] [BOLT] Fix unused variable warnings This patch fixes: bolt/lib/Rewrite/LinuxKernelRewriter.cpp:1664:20: error: unused variable 'TargetAddress' [-Werror,-Wunused-variable] bolt/lib/Rewrite/LinuxKernelRewriter.cpp:1666:20: error: unused variable 'KeyAddress' [-Werror,-Wunused-variable] --- bolt/lib/Rewrite/LinuxKernelRewriter.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bolt/lib/Rewrite/LinuxKernelRewriter.cpp b/bolt/lib/Rewrite/LinuxKernelRewriter.cpp index b028a455a6db..303e8b18fd32 100644 --- a/bolt/lib/Rewrite/LinuxKernelRewriter.cpp +++ b/bolt/lib/Rewrite/LinuxKernelRewriter.cpp @@ -1679,6 +1679,8 @@ Error LinuxKernelRewriter::updateStaticKeysJumpTablePostEmit() { << "\n\tTargetAddress: 0x" << Twine::utohexstr(TargetAddress) << "\n\tKeyAddress: 0x" << Twine::utohexstr(KeyAddress) << '\n'; }); + (void)TargetAddress; + (void)KeyAddress; BinaryFunction *BF = BC.getBinaryFunctionContainingAddress(JumpAddress, -- GitLab From c67ed2f1e12e1b0e16b25606e67b67a47ca848d5 Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Thu, 21 Mar 2024 20:35:08 -0700 Subject: [PATCH 023/306] [SelectionDAG][RISCV] Use TypeSize version of ComputeValueVTs in TargetLowering::LowerCallTo. (#86166) This is needed to support non-intrinsic functions returning tuple types which are represented as structs with scalable vector types in IR. I suspect this may have been broken since https://reviews.llvm.org/D158115 --- .../SelectionDAG/SelectionDAGBuilder.cpp | 8 +- llvm/test/CodeGen/RISCV/rvv/calling-conv.ll | 76 +++++++++++++++++++ 2 files changed, 80 insertions(+), 4 deletions(-) diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 2d63774c75e3..84df98b8a613 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -10500,14 +10500,14 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const { CLI.Ins.clear(); Type *OrigRetTy = CLI.RetTy; SmallVector RetTys; - SmallVector Offsets; + SmallVector Offsets; auto &DL = CLI.DAG.getDataLayout(); - ComputeValueVTs(*this, DL, CLI.RetTy, RetTys, &Offsets, 0); + ComputeValueVTs(*this, DL, CLI.RetTy, RetTys, &Offsets); if (CLI.IsPostTypeLegalization) { // If we are lowering a libcall after legalization, split the return type. SmallVector OldRetTys; - SmallVector OldOffsets; + SmallVector OldOffsets; RetTys.swap(OldRetTys); Offsets.swap(OldOffsets); @@ -10519,7 +10519,7 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const { unsigned RegisterVTByteSZ = RegisterVT.getSizeInBits() / 8; RetTys.append(NumRegs, RegisterVT); for (unsigned j = 0; j != NumRegs; ++j) - Offsets.push_back(Offset + j * RegisterVTByteSZ); + Offsets.push_back(TypeSize::getFixed(Offset + j * RegisterVTByteSZ)); } } diff --git a/llvm/test/CodeGen/RISCV/rvv/calling-conv.ll b/llvm/test/CodeGen/RISCV/rvv/calling-conv.ll index 78385a80b47e..78e8700a9fef 100644 --- a/llvm/test/CodeGen/RISCV/rvv/calling-conv.ll +++ b/llvm/test/CodeGen/RISCV/rvv/calling-conv.ll @@ -86,3 +86,79 @@ define @caller_scalable_vector_split_indirect( @callee_scalable_vector_split_indirect( zeroinitializer, %x) ret %a } + +define {, } @caller_tuple_return() { +; RV32-LABEL: caller_tuple_return: +; RV32: # %bb.0: +; RV32-NEXT: addi sp, sp, -16 +; RV32-NEXT: .cfi_def_cfa_offset 16 +; RV32-NEXT: sw ra, 12(sp) # 4-byte Folded Spill +; RV32-NEXT: .cfi_offset ra, -4 +; RV32-NEXT: call callee_tuple_return +; RV32-NEXT: vmv2r.v v12, v8 +; RV32-NEXT: vmv2r.v v8, v10 +; RV32-NEXT: vmv2r.v v10, v12 +; RV32-NEXT: lw ra, 12(sp) # 4-byte Folded Reload +; RV32-NEXT: addi sp, sp, 16 +; RV32-NEXT: ret +; +; RV64-LABEL: caller_tuple_return: +; RV64: # %bb.0: +; RV64-NEXT: addi sp, sp, -16 +; RV64-NEXT: .cfi_def_cfa_offset 16 +; RV64-NEXT: sd ra, 8(sp) # 8-byte Folded Spill +; RV64-NEXT: .cfi_offset ra, -8 +; RV64-NEXT: call callee_tuple_return +; RV64-NEXT: vmv2r.v v12, v8 +; RV64-NEXT: vmv2r.v v8, v10 +; RV64-NEXT: vmv2r.v v10, v12 +; RV64-NEXT: ld ra, 8(sp) # 8-byte Folded Reload +; RV64-NEXT: addi sp, sp, 16 +; RV64-NEXT: ret + %a = call {, } @callee_tuple_return() + %b = extractvalue {, } %a, 0 + %c = extractvalue {, } %a, 1 + %d = insertvalue {, } poison, %c, 0 + %e = insertvalue {, } %d, %b, 1 + ret {, } %e +} + +declare {, } @callee_tuple_return() + +define void @caller_tuple_argument({, } %x) { +; RV32-LABEL: caller_tuple_argument: +; RV32: # %bb.0: +; RV32-NEXT: addi sp, sp, -16 +; RV32-NEXT: .cfi_def_cfa_offset 16 +; RV32-NEXT: sw ra, 12(sp) # 4-byte Folded Spill +; RV32-NEXT: .cfi_offset ra, -4 +; RV32-NEXT: vmv2r.v v12, v8 +; RV32-NEXT: vmv2r.v v8, v10 +; RV32-NEXT: vmv2r.v v10, v12 +; RV32-NEXT: call callee_tuple_argument +; RV32-NEXT: lw ra, 12(sp) # 4-byte Folded Reload +; RV32-NEXT: addi sp, sp, 16 +; RV32-NEXT: ret +; +; RV64-LABEL: caller_tuple_argument: +; RV64: # %bb.0: +; RV64-NEXT: addi sp, sp, -16 +; RV64-NEXT: .cfi_def_cfa_offset 16 +; RV64-NEXT: sd ra, 8(sp) # 8-byte Folded Spill +; RV64-NEXT: .cfi_offset ra, -8 +; RV64-NEXT: vmv2r.v v12, v8 +; RV64-NEXT: vmv2r.v v8, v10 +; RV64-NEXT: vmv2r.v v10, v12 +; RV64-NEXT: call callee_tuple_argument +; RV64-NEXT: ld ra, 8(sp) # 8-byte Folded Reload +; RV64-NEXT: addi sp, sp, 16 +; RV64-NEXT: ret + %a = extractvalue {, } %x, 0 + %b = extractvalue {, } %x, 1 + %c = insertvalue {, } poison, %b, 0 + %d = insertvalue {, } %c, %a, 1 + call void @callee_tuple_argument({, } %d) + ret void +} + +declare void @callee_tuple_argument({, }) -- GitLab From e1a8120a63cdb6c9567b0f68d9a0390e4f5da184 Mon Sep 17 00:00:00 2001 From: Pravin Jagtap Date: Fri, 22 Mar 2024 09:25:06 +0530 Subject: [PATCH 024/306] [AMDGPU] Support double type in atomic optimizer. (#84307) Presently the atomic optimizer supports only 32-bit operations. Plan is to extend the atomic optimizer for 64-bit operations for compute and graphics. This patch extends support for double type for `uniform values` only. Going forward, will extend the support for divergent values. Adding support for divergent values requires extending/legalizing readfirstlane, readlane, writelane, etc ops for 64-bit operations to avoid `bitcast` noise that we have currently. --------- Authored-by: Pravin Jagtap --- .../Target/AMDGPU/AMDGPUAtomicOptimizer.cpp | 11 +- .../AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll | 274 +- .../CodeGen/AMDGPU/fp64-atomics-gfx90a.ll | 270 +- .../AMDGPU/global_atomic_optimizer_fp_rtn.ll | 560 ++ .../global_atomics_optimizer_fp_no_rtn.ll | 420 ++ .../AMDGPU/global_atomics_scan_fadd.ll | 5578 +++++++++++++++++ .../AMDGPU/global_atomics_scan_fmax.ll | 3960 ++++++++++++ .../AMDGPU/global_atomics_scan_fmin.ll | 3960 ++++++++++++ .../AMDGPU/global_atomics_scan_fsub.ll | 5576 ++++++++++++++++ 9 files changed, 20497 insertions(+), 112 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp index 9ba74a23e8af..dbb3de76b4dd 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp @@ -209,8 +209,9 @@ void AMDGPUAtomicOptimizerImpl::visitAtomicRMWInst(AtomicRMWInst &I) { break; } - // Only 32-bit floating point atomic ops are supported. - if (AtomicRMWInst::isFPOperation(Op) && !I.getType()->isFloatTy()) { + // Only 32 and 64 bit floating point atomic ops are supported. + if (AtomicRMWInst::isFPOperation(Op) && + !(I.getType()->isFloatTy() || I.getType()->isDoubleTy())) { return; } @@ -920,8 +921,10 @@ void AMDGPUAtomicOptimizerImpl::optimizeAtomic(Instruction &I, Value *BroadcastI = nullptr; if (TyBitWidth == 64) { - Value *const ExtractLo = B.CreateTrunc(PHI, Int32Ty); - Value *const ExtractHi = B.CreateTrunc(B.CreateLShr(PHI, 32), Int32Ty); + Value *CastedPhi = B.CreateBitCast(PHI, IntNTy); + Value *const ExtractLo = B.CreateTrunc(CastedPhi, Int32Ty); + Value *const ExtractHi = + B.CreateTrunc(B.CreateLShr(CastedPhi, 32), Int32Ty); CallInst *const ReadFirstLaneLo = B.CreateIntrinsic(Intrinsic::amdgcn_readfirstlane, {}, ExtractLo); CallInst *const ReadFirstLaneHi = diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll index 255c6dedbd6e..1a76f8cf87ff 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll @@ -1090,18 +1090,29 @@ main_body: define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat(ptr addrspace(1) %ptr) #1 { ; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat: ; GFX90A: ; %bb.0: ; %main_body +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: s_mov_b32 s4, s3 +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s4, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB39_3 +; GFX90A-NEXT: ; %bb.1: ; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX90A-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 +; GFX90A-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0 ; GFX90A-NEXT: s_mov_b64 s[2:3], 0 -; GFX90A-NEXT: v_mov_b32_e32 v4, 0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX90A-NEXT: v_mov_b32_e32 v6, 0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1] -; GFX90A-NEXT: .LBB39_1: ; %atomicrmw.start +; GFX90A-NEXT: .LBB39_2: ; %atomicrmw.start ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], 4.0 +; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] ; GFX90A-NEXT: buffer_wbl2 -; GFX90A-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX90A-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: buffer_invl2 ; GFX90A-NEXT: buffer_wbinvl1_vol @@ -1109,20 +1120,31 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat(ptr addrspace(1) %pt ; GFX90A-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1] ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[2:3] -; GFX90A-NEXT: s_cbranch_execnz .LBB39_1 -; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX90A-NEXT: s_cbranch_execnz .LBB39_2 +; GFX90A-NEXT: .LBB39_3: ; GFX90A-NEXT: s_endpgm ; ; GFX940-LABEL: global_atomic_fadd_f64_noret_pat: ; GFX940: ; %bb.0: ; %main_body +; GFX940-NEXT: s_mov_b64 s[2:3], exec +; GFX940-NEXT: s_mov_b32 s4, s3 +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v0, s4, v0 +; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX940-NEXT: s_cbranch_execz .LBB39_2 +; GFX940-NEXT: ; %bb.1: ; GFX940-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 -; GFX940-NEXT: v_mov_b64_e32 v[0:1], 4.0 +; GFX940-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX940-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 +; GFX940-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX940-NEXT: v_mov_b32_e32 v2, 0 ; GFX940-NEXT: buffer_wbl2 sc0 sc1 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) ; GFX940-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] sc1 ; GFX940-NEXT: s_waitcnt vmcnt(0) ; GFX940-NEXT: buffer_inv sc0 sc1 +; GFX940-NEXT: .LBB39_2: ; GFX940-NEXT: s_endpgm main_body: %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 seq_cst @@ -1132,26 +1154,47 @@ main_body: define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_agent(ptr addrspace(1) %ptr) #1 { ; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat_agent: ; GFX90A: ; %bb.0: ; %main_body +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: s_mov_b32 s4, s3 +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s4, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB40_2 +; GFX90A-NEXT: ; %bb.1: ; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 -; GFX90A-NEXT: v_mov_b32_e32 v0, 0 -; GFX90A-NEXT: v_mov_b32_e32 v1, 0x40100000 +; GFX90A-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 +; GFX90A-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX90A-NEXT: v_mov_b32_e32 v2, 0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: buffer_wbinvl1_vol +; GFX90A-NEXT: .LBB40_2: ; GFX90A-NEXT: s_endpgm ; ; GFX940-LABEL: global_atomic_fadd_f64_noret_pat_agent: ; GFX940: ; %bb.0: ; %main_body +; GFX940-NEXT: s_mov_b64 s[2:3], exec +; GFX940-NEXT: s_mov_b32 s4, s3 +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v0, s4, v0 +; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX940-NEXT: s_cbranch_execz .LBB40_2 +; GFX940-NEXT: ; %bb.1: ; GFX940-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 -; GFX940-NEXT: v_mov_b64_e32 v[0:1], 4.0 +; GFX940-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX940-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 +; GFX940-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX940-NEXT: v_mov_b32_e32 v2, 0 ; GFX940-NEXT: buffer_wbl2 sc1 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) ; GFX940-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] ; GFX940-NEXT: s_waitcnt vmcnt(0) ; GFX940-NEXT: buffer_inv sc1 +; GFX940-NEXT: .LBB40_2: ; GFX940-NEXT: s_endpgm main_body: %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("agent") seq_cst @@ -1161,18 +1204,29 @@ main_body: define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_system(ptr addrspace(1) %ptr) #1 { ; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat_system: ; GFX90A: ; %bb.0: ; %main_body +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: s_mov_b32 s4, s3 +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s4, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB41_3 +; GFX90A-NEXT: ; %bb.1: ; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX90A-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 +; GFX90A-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0 ; GFX90A-NEXT: s_mov_b64 s[2:3], 0 -; GFX90A-NEXT: v_mov_b32_e32 v4, 0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX90A-NEXT: v_mov_b32_e32 v6, 0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1] -; GFX90A-NEXT: .LBB41_1: ; %atomicrmw.start +; GFX90A-NEXT: .LBB41_2: ; %atomicrmw.start ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], 4.0 +; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] ; GFX90A-NEXT: buffer_wbl2 -; GFX90A-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX90A-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: buffer_invl2 ; GFX90A-NEXT: buffer_wbinvl1_vol @@ -1180,20 +1234,31 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_system(ptr addrspace ; GFX90A-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1] ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[2:3] -; GFX90A-NEXT: s_cbranch_execnz .LBB41_1 -; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX90A-NEXT: s_cbranch_execnz .LBB41_2 +; GFX90A-NEXT: .LBB41_3: ; GFX90A-NEXT: s_endpgm ; ; GFX940-LABEL: global_atomic_fadd_f64_noret_pat_system: ; GFX940: ; %bb.0: ; %main_body +; GFX940-NEXT: s_mov_b64 s[2:3], exec +; GFX940-NEXT: s_mov_b32 s4, s3 +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v0, s4, v0 +; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX940-NEXT: s_cbranch_execz .LBB41_2 +; GFX940-NEXT: ; %bb.1: ; GFX940-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 -; GFX940-NEXT: v_mov_b64_e32 v[0:1], 4.0 +; GFX940-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX940-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 +; GFX940-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX940-NEXT: v_mov_b32_e32 v2, 0 ; GFX940-NEXT: buffer_wbl2 sc0 sc1 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) ; GFX940-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] sc1 ; GFX940-NEXT: s_waitcnt vmcnt(0) ; GFX940-NEXT: buffer_inv sc0 sc1 +; GFX940-NEXT: .LBB41_2: ; GFX940-NEXT: s_endpgm main_body: %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("one-as") seq_cst @@ -1203,26 +1268,47 @@ main_body: define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_flush(ptr addrspace(1) %ptr) #0 { ; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat_flush: ; GFX90A: ; %bb.0: ; %main_body +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: s_mov_b32 s4, s3 +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s4, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB42_2 +; GFX90A-NEXT: ; %bb.1: ; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 -; GFX90A-NEXT: v_mov_b32_e32 v0, 0 -; GFX90A-NEXT: v_mov_b32_e32 v1, 0x40100000 +; GFX90A-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 +; GFX90A-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX90A-NEXT: v_mov_b32_e32 v2, 0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: buffer_wbinvl1_vol +; GFX90A-NEXT: .LBB42_2: ; GFX90A-NEXT: s_endpgm ; ; GFX940-LABEL: global_atomic_fadd_f64_noret_pat_flush: ; GFX940: ; %bb.0: ; %main_body +; GFX940-NEXT: s_mov_b64 s[2:3], exec +; GFX940-NEXT: s_mov_b32 s4, s3 +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v0, s4, v0 +; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX940-NEXT: s_cbranch_execz .LBB42_2 +; GFX940-NEXT: ; %bb.1: ; GFX940-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 -; GFX940-NEXT: v_mov_b64_e32 v[0:1], 4.0 +; GFX940-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX940-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 +; GFX940-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX940-NEXT: v_mov_b32_e32 v2, 0 ; GFX940-NEXT: buffer_wbl2 sc1 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) ; GFX940-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] ; GFX940-NEXT: s_waitcnt vmcnt(0) ; GFX940-NEXT: buffer_inv sc1 +; GFX940-NEXT: .LBB42_2: ; GFX940-NEXT: s_endpgm main_body: %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("agent") seq_cst @@ -1394,37 +1480,59 @@ main_body: define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_agent_safe(ptr addrspace(1) %ptr) { ; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat_agent_safe: ; GFX90A: ; %bb.0: ; %main_body +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: s_mov_b32 s4, s3 +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s4, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB49_3 +; GFX90A-NEXT: ; %bb.1: ; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX90A-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 +; GFX90A-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0 ; GFX90A-NEXT: s_mov_b64 s[2:3], 0 -; GFX90A-NEXT: v_mov_b32_e32 v4, 0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX90A-NEXT: v_mov_b32_e32 v6, 0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1] -; GFX90A-NEXT: .LBB49_1: ; %atomicrmw.start +; GFX90A-NEXT: .LBB49_2: ; %atomicrmw.start ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], 4.0 -; GFX90A-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX90A-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: buffer_wbinvl1_vol ; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GFX90A-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1] ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[2:3] -; GFX90A-NEXT: s_cbranch_execnz .LBB49_1 -; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX90A-NEXT: s_cbranch_execnz .LBB49_2 +; GFX90A-NEXT: .LBB49_3: ; GFX90A-NEXT: s_endpgm ; ; GFX940-LABEL: global_atomic_fadd_f64_noret_pat_agent_safe: ; GFX940: ; %bb.0: ; %main_body +; GFX940-NEXT: s_mov_b64 s[2:3], exec +; GFX940-NEXT: s_mov_b32 s4, s3 +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v0, s4, v0 +; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX940-NEXT: s_cbranch_execz .LBB49_2 +; GFX940-NEXT: ; %bb.1: ; GFX940-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 -; GFX940-NEXT: v_mov_b64_e32 v[0:1], 4.0 +; GFX940-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX940-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 +; GFX940-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX940-NEXT: v_mov_b32_e32 v2, 0 ; GFX940-NEXT: buffer_wbl2 sc1 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) ; GFX940-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] ; GFX940-NEXT: s_waitcnt vmcnt(0) ; GFX940-NEXT: buffer_inv sc1 +; GFX940-NEXT: .LBB49_2: ; GFX940-NEXT: s_endpgm main_body: %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("agent") seq_cst @@ -1866,23 +1974,44 @@ main_body: define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat(ptr addrspace(3) %ptr) #1 { ; GFX90A-LABEL: local_atomic_fadd_f64_noret_pat: ; GFX90A: ; %bb.0: ; %main_body +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: s_mov_b32 s4, s3 +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s4, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB65_2 +; GFX90A-NEXT: ; %bb.1: ; GFX90A-NEXT: s_load_dword s0, s[0:1], 0x24 -; GFX90A-NEXT: v_mov_b32_e32 v0, 0 -; GFX90A-NEXT: v_mov_b32_e32 v1, 0x40100000 +; GFX90A-NEXT: s_bcnt1_i32_b64 s1, s[2:3] +; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s1 +; GFX90A-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: v_mov_b32_e32 v2, s0 ; GFX90A-NEXT: ds_add_f64 v2, v[0:1] ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: .LBB65_2: ; GFX90A-NEXT: s_endpgm ; ; GFX940-LABEL: local_atomic_fadd_f64_noret_pat: ; GFX940: ; %bb.0: ; %main_body +; GFX940-NEXT: s_mov_b64 s[2:3], exec +; GFX940-NEXT: s_mov_b32 s4, s3 +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v0, s4, v0 +; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX940-NEXT: s_cbranch_execz .LBB65_2 +; GFX940-NEXT: ; %bb.1: ; GFX940-NEXT: s_load_dword s0, s[0:1], 0x24 -; GFX940-NEXT: v_mov_b64_e32 v[0:1], 4.0 +; GFX940-NEXT: s_bcnt1_i32_b64 s1, s[2:3] +; GFX940-NEXT: v_cvt_f64_u32_e32 v[0:1], s1 +; GFX940-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) ; GFX940-NEXT: v_mov_b32_e32 v2, s0 ; GFX940-NEXT: ds_add_f64 v2, v[0:1] ; GFX940-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-NEXT: .LBB65_2: ; GFX940-NEXT: s_endpgm main_body: %ret = atomicrmw fadd ptr addrspace(3) %ptr, double 4.0 seq_cst @@ -1892,23 +2021,44 @@ main_body: define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush(ptr addrspace(3) %ptr) #0 { ; GFX90A-LABEL: local_atomic_fadd_f64_noret_pat_flush: ; GFX90A: ; %bb.0: ; %main_body +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: s_mov_b32 s4, s3 +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s4, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB66_2 +; GFX90A-NEXT: ; %bb.1: ; GFX90A-NEXT: s_load_dword s0, s[0:1], 0x24 -; GFX90A-NEXT: v_mov_b32_e32 v0, 0 -; GFX90A-NEXT: v_mov_b32_e32 v1, 0x40100000 +; GFX90A-NEXT: s_bcnt1_i32_b64 s1, s[2:3] +; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s1 +; GFX90A-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: v_mov_b32_e32 v2, s0 ; GFX90A-NEXT: ds_add_f64 v2, v[0:1] ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: .LBB66_2: ; GFX90A-NEXT: s_endpgm ; ; GFX940-LABEL: local_atomic_fadd_f64_noret_pat_flush: ; GFX940: ; %bb.0: ; %main_body +; GFX940-NEXT: s_mov_b64 s[2:3], exec +; GFX940-NEXT: s_mov_b32 s4, s3 +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v0, s4, v0 +; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX940-NEXT: s_cbranch_execz .LBB66_2 +; GFX940-NEXT: ; %bb.1: ; GFX940-NEXT: s_load_dword s0, s[0:1], 0x24 -; GFX940-NEXT: v_mov_b64_e32 v[0:1], 4.0 +; GFX940-NEXT: s_bcnt1_i32_b64 s1, s[2:3] +; GFX940-NEXT: v_cvt_f64_u32_e32 v[0:1], s1 +; GFX940-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) ; GFX940-NEXT: v_mov_b32_e32 v2, s0 ; GFX940-NEXT: ds_add_f64 v2, v[0:1] ; GFX940-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-NEXT: .LBB66_2: ; GFX940-NEXT: s_endpgm main_body: %ret = atomicrmw fadd ptr addrspace(3) %ptr, double 4.0 seq_cst @@ -1918,44 +2068,66 @@ main_body: define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush_safe(ptr addrspace(3) %ptr) #4 { ; GFX90A-LABEL: local_atomic_fadd_f64_noret_pat_flush_safe: ; GFX90A: ; %bb.0: ; %main_body +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: s_mov_b32 s4, s3 +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s4, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB67_3 +; GFX90A-NEXT: ; %bb.1: ; GFX90A-NEXT: s_load_dword s0, s[0:1], 0x24 +; GFX90A-NEXT: s_bcnt1_i32_b64 s1, s[2:3] +; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s1 +; GFX90A-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_mov_b32_e32 v2, s0 -; GFX90A-NEXT: ds_read_b64 v[0:1], v2 +; GFX90A-NEXT: v_mov_b32_e32 v4, s0 +; GFX90A-NEXT: ds_read_b64 v[2:3], v4 ; GFX90A-NEXT: s_mov_b64 s[0:1], 0 -; GFX90A-NEXT: .LBB67_1: ; %atomicrmw.start +; GFX90A-NEXT: .LBB67_2: ; %atomicrmw.start ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_add_f64 v[4:5], v[0:1], 4.0 -; GFX90A-NEXT: ds_cmpst_rtn_b64 v[4:5], v2, v[0:1], v[4:5] +; GFX90A-NEXT: v_add_f64 v[6:7], v[2:3], v[0:1] +; GFX90A-NEXT: ds_cmpst_rtn_b64 v[6:7], v4, v[2:3], v[6:7] ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[0:1] +; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3] ; GFX90A-NEXT: s_or_b64 s[0:1], vcc, s[0:1] -; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1] +; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[6:7], v[6:7] op_sel:[0,1] ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[0:1] -; GFX90A-NEXT: s_cbranch_execnz .LBB67_1 -; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX90A-NEXT: s_cbranch_execnz .LBB67_2 +; GFX90A-NEXT: .LBB67_3: ; GFX90A-NEXT: s_endpgm ; ; GFX940-LABEL: local_atomic_fadd_f64_noret_pat_flush_safe: ; GFX940: ; %bb.0: ; %main_body +; GFX940-NEXT: s_mov_b64 s[2:3], exec +; GFX940-NEXT: s_mov_b32 s4, s3 +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v0, s4, v0 +; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX940-NEXT: s_cbranch_execz .LBB67_3 +; GFX940-NEXT: ; %bb.1: ; GFX940-NEXT: s_load_dword s0, s[0:1], 0x24 +; GFX940-NEXT: s_bcnt1_i32_b64 s1, s[2:3] +; GFX940-NEXT: v_cvt_f64_u32_e32 v[0:1], s1 +; GFX940-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) -; GFX940-NEXT: v_mov_b32_e32 v2, s0 -; GFX940-NEXT: ds_read_b64 v[0:1], v2 +; GFX940-NEXT: v_mov_b32_e32 v4, s0 +; GFX940-NEXT: ds_read_b64 v[2:3], v4 ; GFX940-NEXT: s_mov_b64 s[0:1], 0 -; GFX940-NEXT: .LBB67_1: ; %atomicrmw.start +; GFX940-NEXT: .LBB67_2: ; %atomicrmw.start ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) -; GFX940-NEXT: v_add_f64 v[4:5], v[0:1], 4.0 -; GFX940-NEXT: ds_cmpst_rtn_b64 v[4:5], v2, v[0:1], v[4:5] +; GFX940-NEXT: v_add_f64 v[6:7], v[2:3], v[0:1] +; GFX940-NEXT: ds_cmpst_rtn_b64 v[6:7], v4, v[2:3], v[6:7] ; GFX940-NEXT: s_waitcnt lgkmcnt(0) -; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[0:1] +; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3] ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1] -; GFX940-NEXT: v_mov_b64_e32 v[0:1], v[4:5] +; GFX940-NEXT: v_mov_b64_e32 v[2:3], v[6:7] ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1] -; GFX940-NEXT: s_cbranch_execnz .LBB67_1 -; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX940-NEXT: s_cbranch_execnz .LBB67_2 +; GFX940-NEXT: .LBB67_3: ; GFX940-NEXT: s_endpgm main_body: %ret = atomicrmw fadd ptr addrspace(3) %ptr, double 4.0 seq_cst diff --git a/llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll b/llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll index 767d347bcfaa..a948fab8f1c1 100644 --- a/llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll +++ b/llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll @@ -1181,18 +1181,28 @@ main_body: define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat(ptr addrspace(1) %ptr) #1 { ; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat: ; GFX90A: ; %bb.0: ; %main_body +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB42_3 +; GFX90A-NEXT: ; %bb.1: ; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX90A-NEXT: s_bcnt1_i32_b64 s6, s[2:3] +; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s6 ; GFX90A-NEXT: s_mov_b64 s[2:3], 0 -; GFX90A-NEXT: v_mov_b32_e32 v4, 0 +; GFX90A-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX90A-NEXT: v_mov_b32_e32 v6, 0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1] -; GFX90A-NEXT: .LBB42_1: ; %atomicrmw.start +; GFX90A-NEXT: .LBB42_2: ; %atomicrmw.start ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], 4.0 +; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] ; GFX90A-NEXT: buffer_wbl2 -; GFX90A-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX90A-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: buffer_invl2 ; GFX90A-NEXT: buffer_wbinvl1_vol @@ -1200,20 +1210,30 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat(ptr addrspace(1) %pt ; GFX90A-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1] ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[2:3] -; GFX90A-NEXT: s_cbranch_execnz .LBB42_1 -; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX90A-NEXT: s_cbranch_execnz .LBB42_2 +; GFX90A-NEXT: .LBB42_3: ; GFX90A-NEXT: s_endpgm ; ; GFX940-LABEL: global_atomic_fadd_f64_noret_pat: ; GFX940: ; %bb.0: ; %main_body +; GFX940-NEXT: s_mov_b64 s[2:3], exec +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX940-NEXT: s_cbranch_execz .LBB42_2 +; GFX940-NEXT: ; %bb.1: ; GFX940-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX940-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX940-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 ; GFX940-NEXT: v_mov_b32_e32 v2, 0 -; GFX940-NEXT: v_mov_b64_e32 v[0:1], 4.0 +; GFX940-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX940-NEXT: buffer_wbl2 sc0 sc1 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) ; GFX940-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] sc1 ; GFX940-NEXT: s_waitcnt vmcnt(0) ; GFX940-NEXT: buffer_inv sc0 sc1 +; GFX940-NEXT: .LBB42_2: ; GFX940-NEXT: s_endpgm main_body: %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 seq_cst @@ -1223,26 +1243,45 @@ main_body: define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_agent(ptr addrspace(1) %ptr) #1 { ; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat_agent: ; GFX90A: ; %bb.0: ; %main_body +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB43_2 +; GFX90A-NEXT: ; %bb.1: ; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 -; GFX90A-NEXT: v_mov_b32_e32 v0, 0 +; GFX90A-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 ; GFX90A-NEXT: v_mov_b32_e32 v2, 0 -; GFX90A-NEXT: v_mov_b32_e32 v1, 0x40100000 +; GFX90A-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: buffer_wbinvl1_vol +; GFX90A-NEXT: .LBB43_2: ; GFX90A-NEXT: s_endpgm ; ; GFX940-LABEL: global_atomic_fadd_f64_noret_pat_agent: ; GFX940: ; %bb.0: ; %main_body +; GFX940-NEXT: s_mov_b64 s[2:3], exec +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX940-NEXT: s_cbranch_execz .LBB43_2 +; GFX940-NEXT: ; %bb.1: ; GFX940-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX940-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX940-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 ; GFX940-NEXT: v_mov_b32_e32 v2, 0 -; GFX940-NEXT: v_mov_b64_e32 v[0:1], 4.0 +; GFX940-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX940-NEXT: buffer_wbl2 sc1 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) ; GFX940-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] ; GFX940-NEXT: s_waitcnt vmcnt(0) ; GFX940-NEXT: buffer_inv sc1 +; GFX940-NEXT: .LBB43_2: ; GFX940-NEXT: s_endpgm main_body: %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("agent") seq_cst @@ -1252,18 +1291,28 @@ main_body: define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_system(ptr addrspace(1) %ptr) #1 { ; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat_system: ; GFX90A: ; %bb.0: ; %main_body +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB44_3 +; GFX90A-NEXT: ; %bb.1: ; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX90A-NEXT: s_bcnt1_i32_b64 s6, s[2:3] +; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s6 ; GFX90A-NEXT: s_mov_b64 s[2:3], 0 -; GFX90A-NEXT: v_mov_b32_e32 v4, 0 +; GFX90A-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX90A-NEXT: v_mov_b32_e32 v6, 0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1] -; GFX90A-NEXT: .LBB44_1: ; %atomicrmw.start +; GFX90A-NEXT: .LBB44_2: ; %atomicrmw.start ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], 4.0 +; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] ; GFX90A-NEXT: buffer_wbl2 -; GFX90A-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX90A-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: buffer_invl2 ; GFX90A-NEXT: buffer_wbinvl1_vol @@ -1271,20 +1320,30 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_system(ptr addrspace ; GFX90A-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1] ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[2:3] -; GFX90A-NEXT: s_cbranch_execnz .LBB44_1 -; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX90A-NEXT: s_cbranch_execnz .LBB44_2 +; GFX90A-NEXT: .LBB44_3: ; GFX90A-NEXT: s_endpgm ; ; GFX940-LABEL: global_atomic_fadd_f64_noret_pat_system: ; GFX940: ; %bb.0: ; %main_body +; GFX940-NEXT: s_mov_b64 s[2:3], exec +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX940-NEXT: s_cbranch_execz .LBB44_2 +; GFX940-NEXT: ; %bb.1: ; GFX940-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX940-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX940-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 ; GFX940-NEXT: v_mov_b32_e32 v2, 0 -; GFX940-NEXT: v_mov_b64_e32 v[0:1], 4.0 +; GFX940-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX940-NEXT: buffer_wbl2 sc0 sc1 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) ; GFX940-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] sc1 ; GFX940-NEXT: s_waitcnt vmcnt(0) ; GFX940-NEXT: buffer_inv sc0 sc1 +; GFX940-NEXT: .LBB44_2: ; GFX940-NEXT: s_endpgm main_body: %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("one-as") seq_cst @@ -1294,26 +1353,45 @@ main_body: define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_flush(ptr addrspace(1) %ptr) #0 { ; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat_flush: ; GFX90A: ; %bb.0: ; %main_body +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB45_2 +; GFX90A-NEXT: ; %bb.1: ; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 -; GFX90A-NEXT: v_mov_b32_e32 v0, 0 +; GFX90A-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 ; GFX90A-NEXT: v_mov_b32_e32 v2, 0 -; GFX90A-NEXT: v_mov_b32_e32 v1, 0x40100000 +; GFX90A-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: buffer_wbinvl1_vol +; GFX90A-NEXT: .LBB45_2: ; GFX90A-NEXT: s_endpgm ; ; GFX940-LABEL: global_atomic_fadd_f64_noret_pat_flush: ; GFX940: ; %bb.0: ; %main_body +; GFX940-NEXT: s_mov_b64 s[2:3], exec +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX940-NEXT: s_cbranch_execz .LBB45_2 +; GFX940-NEXT: ; %bb.1: ; GFX940-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX940-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX940-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 ; GFX940-NEXT: v_mov_b32_e32 v2, 0 -; GFX940-NEXT: v_mov_b64_e32 v[0:1], 4.0 +; GFX940-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX940-NEXT: buffer_wbl2 sc1 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) ; GFX940-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] ; GFX940-NEXT: s_waitcnt vmcnt(0) ; GFX940-NEXT: buffer_inv sc1 +; GFX940-NEXT: .LBB45_2: ; GFX940-NEXT: s_endpgm main_body: %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("agent") seq_cst @@ -1485,37 +1563,57 @@ main_body: define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_agent_safe(ptr addrspace(1) %ptr) { ; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat_agent_safe: ; GFX90A: ; %bb.0: ; %main_body +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB52_3 +; GFX90A-NEXT: ; %bb.1: ; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX90A-NEXT: s_bcnt1_i32_b64 s6, s[2:3] +; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s6 ; GFX90A-NEXT: s_mov_b64 s[2:3], 0 -; GFX90A-NEXT: v_mov_b32_e32 v4, 0 +; GFX90A-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX90A-NEXT: v_mov_b32_e32 v6, 0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1] -; GFX90A-NEXT: .LBB52_1: ; %atomicrmw.start +; GFX90A-NEXT: .LBB52_2: ; %atomicrmw.start ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], 4.0 -; GFX90A-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX90A-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: buffer_wbinvl1_vol ; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GFX90A-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1] ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[2:3] -; GFX90A-NEXT: s_cbranch_execnz .LBB52_1 -; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX90A-NEXT: s_cbranch_execnz .LBB52_2 +; GFX90A-NEXT: .LBB52_3: ; GFX90A-NEXT: s_endpgm ; ; GFX940-LABEL: global_atomic_fadd_f64_noret_pat_agent_safe: ; GFX940: ; %bb.0: ; %main_body +; GFX940-NEXT: s_mov_b64 s[2:3], exec +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX940-NEXT: s_cbranch_execz .LBB52_2 +; GFX940-NEXT: ; %bb.1: ; GFX940-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX940-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX940-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 ; GFX940-NEXT: v_mov_b32_e32 v2, 0 -; GFX940-NEXT: v_mov_b64_e32 v[0:1], 4.0 +; GFX940-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX940-NEXT: buffer_wbl2 sc1 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) ; GFX940-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] ; GFX940-NEXT: s_waitcnt vmcnt(0) ; GFX940-NEXT: buffer_inv sc1 +; GFX940-NEXT: .LBB52_2: ; GFX940-NEXT: s_endpgm main_body: %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("agent") seq_cst @@ -2020,23 +2118,42 @@ main_body: define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat(ptr addrspace(3) %ptr) #1 { ; GFX90A-LABEL: local_atomic_fadd_f64_noret_pat: ; GFX90A: ; %bb.0: ; %main_body +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB70_2 +; GFX90A-NEXT: ; %bb.1: ; GFX90A-NEXT: s_load_dword s0, s[0:1], 0x24 -; GFX90A-NEXT: v_mov_b32_e32 v0, 0 -; GFX90A-NEXT: v_mov_b32_e32 v1, 0x40100000 +; GFX90A-NEXT: s_bcnt1_i32_b64 s1, s[2:3] +; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s1 +; GFX90A-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: v_mov_b32_e32 v2, s0 ; GFX90A-NEXT: ds_add_f64 v2, v[0:1] ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: .LBB70_2: ; GFX90A-NEXT: s_endpgm ; ; GFX940-LABEL: local_atomic_fadd_f64_noret_pat: ; GFX940: ; %bb.0: ; %main_body +; GFX940-NEXT: s_mov_b64 s[2:3], exec +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX940-NEXT: s_cbranch_execz .LBB70_2 +; GFX940-NEXT: ; %bb.1: ; GFX940-NEXT: s_load_dword s0, s[0:1], 0x24 -; GFX940-NEXT: v_mov_b64_e32 v[0:1], 4.0 +; GFX940-NEXT: s_bcnt1_i32_b64 s1, s[2:3] +; GFX940-NEXT: v_cvt_f64_u32_e32 v[0:1], s1 +; GFX940-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) ; GFX940-NEXT: v_mov_b32_e32 v2, s0 ; GFX940-NEXT: ds_add_f64 v2, v[0:1] ; GFX940-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-NEXT: .LBB70_2: ; GFX940-NEXT: s_endpgm main_body: %ret = atomicrmw fadd ptr addrspace(3) %ptr, double 4.0 seq_cst @@ -2046,23 +2163,42 @@ main_body: define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush(ptr addrspace(3) %ptr) #0 { ; GFX90A-LABEL: local_atomic_fadd_f64_noret_pat_flush: ; GFX90A: ; %bb.0: ; %main_body +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB71_2 +; GFX90A-NEXT: ; %bb.1: ; GFX90A-NEXT: s_load_dword s0, s[0:1], 0x24 -; GFX90A-NEXT: v_mov_b32_e32 v0, 0 -; GFX90A-NEXT: v_mov_b32_e32 v1, 0x40100000 +; GFX90A-NEXT: s_bcnt1_i32_b64 s1, s[2:3] +; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s1 +; GFX90A-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: v_mov_b32_e32 v2, s0 ; GFX90A-NEXT: ds_add_f64 v2, v[0:1] ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: .LBB71_2: ; GFX90A-NEXT: s_endpgm ; ; GFX940-LABEL: local_atomic_fadd_f64_noret_pat_flush: ; GFX940: ; %bb.0: ; %main_body +; GFX940-NEXT: s_mov_b64 s[2:3], exec +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX940-NEXT: s_cbranch_execz .LBB71_2 +; GFX940-NEXT: ; %bb.1: ; GFX940-NEXT: s_load_dword s0, s[0:1], 0x24 -; GFX940-NEXT: v_mov_b64_e32 v[0:1], 4.0 +; GFX940-NEXT: s_bcnt1_i32_b64 s1, s[2:3] +; GFX940-NEXT: v_cvt_f64_u32_e32 v[0:1], s1 +; GFX940-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) ; GFX940-NEXT: v_mov_b32_e32 v2, s0 ; GFX940-NEXT: ds_add_f64 v2, v[0:1] ; GFX940-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-NEXT: .LBB71_2: ; GFX940-NEXT: s_endpgm main_body: %ret = atomicrmw fadd ptr addrspace(3) %ptr, double 4.0 seq_cst @@ -2072,46 +2208,66 @@ main_body: define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush_safe(ptr addrspace(3) %ptr) #4 { ; GFX90A-LABEL: local_atomic_fadd_f64_noret_pat_flush_safe: ; GFX90A: ; %bb.0: ; %main_body -; GFX90A-NEXT: s_load_dword s2, s[0:1], 0x24 -; GFX90A-NEXT: s_mov_b64 s[0:1], 0 +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB72_3 +; GFX90A-NEXT: ; %bb.1: +; GFX90A-NEXT: s_load_dword s4, s[0:1], 0x24 +; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[2:3] ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_mov_b32_e32 v0, s2 -; GFX90A-NEXT: ds_read_b64 v[0:1], v0 -; GFX90A-NEXT: v_mov_b32_e32 v2, s2 -; GFX90A-NEXT: .LBB72_1: ; %atomicrmw.start +; GFX90A-NEXT: v_mov_b32_e32 v0, s4 +; GFX90A-NEXT: ds_read_b64 v[2:3], v0 +; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX90A-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 +; GFX90A-NEXT: s_mov_b64 s[0:1], 0 +; GFX90A-NEXT: v_mov_b32_e32 v4, s4 +; GFX90A-NEXT: .LBB72_2: ; %atomicrmw.start ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_add_f64 v[4:5], v[0:1], 4.0 -; GFX90A-NEXT: ds_cmpst_rtn_b64 v[4:5], v2, v[0:1], v[4:5] +; GFX90A-NEXT: v_add_f64 v[6:7], v[2:3], v[0:1] +; GFX90A-NEXT: ds_cmpst_rtn_b64 v[6:7], v4, v[2:3], v[6:7] ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[0:1] +; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3] ; GFX90A-NEXT: s_or_b64 s[0:1], vcc, s[0:1] -; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1] +; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[6:7], v[6:7] op_sel:[0,1] ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[0:1] -; GFX90A-NEXT: s_cbranch_execnz .LBB72_1 -; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX90A-NEXT: s_cbranch_execnz .LBB72_2 +; GFX90A-NEXT: .LBB72_3: ; GFX90A-NEXT: s_endpgm ; ; GFX940-LABEL: local_atomic_fadd_f64_noret_pat_flush_safe: ; GFX940: ; %bb.0: ; %main_body -; GFX940-NEXT: s_load_dword s2, s[0:1], 0x24 -; GFX940-NEXT: s_mov_b64 s[0:1], 0 +; GFX940-NEXT: s_mov_b64 s[2:3], exec +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX940-NEXT: s_cbranch_execz .LBB72_3 +; GFX940-NEXT: ; %bb.1: +; GFX940-NEXT: s_load_dword s4, s[0:1], 0x24 +; GFX940-NEXT: s_bcnt1_i32_b64 s0, s[2:3] ; GFX940-NEXT: s_waitcnt lgkmcnt(0) -; GFX940-NEXT: v_mov_b32_e32 v0, s2 -; GFX940-NEXT: ds_read_b64 v[0:1], v0 -; GFX940-NEXT: v_mov_b32_e32 v2, s2 -; GFX940-NEXT: .LBB72_1: ; %atomicrmw.start +; GFX940-NEXT: v_mov_b32_e32 v0, s4 +; GFX940-NEXT: ds_read_b64 v[2:3], v0 +; GFX940-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX940-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 +; GFX940-NEXT: s_mov_b64 s[0:1], 0 +; GFX940-NEXT: v_mov_b32_e32 v4, s4 +; GFX940-NEXT: .LBB72_2: ; %atomicrmw.start ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) -; GFX940-NEXT: v_add_f64 v[4:5], v[0:1], 4.0 -; GFX940-NEXT: ds_cmpst_rtn_b64 v[4:5], v2, v[0:1], v[4:5] +; GFX940-NEXT: v_add_f64 v[6:7], v[2:3], v[0:1] +; GFX940-NEXT: ds_cmpst_rtn_b64 v[6:7], v4, v[2:3], v[6:7] ; GFX940-NEXT: s_waitcnt lgkmcnt(0) -; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[0:1] +; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3] ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1] -; GFX940-NEXT: v_mov_b64_e32 v[0:1], v[4:5] +; GFX940-NEXT: v_mov_b64_e32 v[2:3], v[6:7] ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1] -; GFX940-NEXT: s_cbranch_execnz .LBB72_1 -; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX940-NEXT: s_cbranch_execnz .LBB72_2 +; GFX940-NEXT: .LBB72_3: ; GFX940-NEXT: s_endpgm main_body: %ret = atomicrmw fadd ptr addrspace(3) %ptr, double 4.0 seq_cst diff --git a/llvm/test/CodeGen/AMDGPU/global_atomic_optimizer_fp_rtn.ll b/llvm/test/CodeGen/AMDGPU/global_atomic_optimizer_fp_rtn.ll index b7a91f6fa96d..b71728096093 100644 --- a/llvm/test/CodeGen/AMDGPU/global_atomic_optimizer_fp_rtn.ll +++ b/llvm/test/CodeGen/AMDGPU/global_atomic_optimizer_fp_rtn.ll @@ -1058,6 +1058,566 @@ define amdgpu_ps float @global_atomic_fadd_div_address_div_value_system_scope_st ret float %result } +define amdgpu_ps double @global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) inreg %ptr, double inreg %val) #0 { +; IR-LABEL: @global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe( +; IR-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() +; IR-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP30:%.*]] +; IR: 2: +; IR-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) +; IR-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) +; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) +; IR-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) +; IR-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-NEXT: [[TMP11:%.*]] = uitofp i32 [[TMP10]] to double +; IR-NEXT: [[TMP12:%.*]] = fmul double [[VAL:%.*]], [[TMP11]] +; IR-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR: 14: +; IR-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("agent") monotonic, align 4 +; IR-NEXT: br label [[TMP16]] +; IR: 16: +; IR-NEXT: [[TMP17:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ] +; IR-NEXT: [[TMP18:%.*]] = bitcast double [[TMP17]] to i64 +; IR-NEXT: [[TMP19:%.*]] = trunc i64 [[TMP18]] to i32 +; IR-NEXT: [[TMP20:%.*]] = lshr i64 [[TMP18]], 32 +; IR-NEXT: [[TMP21:%.*]] = trunc i64 [[TMP20]] to i32 +; IR-NEXT: [[TMP22:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP19]]) +; IR-NEXT: [[TMP23:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP21]]) +; IR-NEXT: [[TMP24:%.*]] = insertelement <2 x i32> poison, i32 [[TMP22]], i32 0 +; IR-NEXT: [[TMP25:%.*]] = insertelement <2 x i32> [[TMP24]], i32 [[TMP23]], i32 1 +; IR-NEXT: [[TMP26:%.*]] = bitcast <2 x i32> [[TMP25]] to double +; IR-NEXT: [[TMP27:%.*]] = uitofp i32 [[TMP8]] to double +; IR-NEXT: [[TMP28:%.*]] = fmul double [[VAL]], [[TMP27]] +; IR-NEXT: [[TMP29:%.*]] = fadd double [[TMP26]], [[TMP28]] +; IR-NEXT: br label [[TMP30]] +; IR: 30: +; IR-NEXT: [[TMP31:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP29]], [[TMP16]] ] +; IR-NEXT: ret double [[TMP31]] +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic, align 4 + ret double %result +} + +define amdgpu_ps double @global_atomic_fadd_double_uni_address_div_value_scope_agent_scope_unsafe(ptr addrspace(1) inreg %ptr, double %val) #0 { +; IR-LABEL: @global_atomic_fadd_double_uni_address_div_value_scope_agent_scope_unsafe( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 4 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic, align 4 + ret double %result +} + +define amdgpu_ps double @global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp(ptr addrspace(1) inreg %ptr, double inreg %val) #1 { +; IR-ITERATIVE-LABEL: @global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp( +; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]] +; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP30:%.*]] +; IR-ITERATIVE: 2: +; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-ITERATIVE-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR-ITERATIVE: 14: +; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("one-as") monotonic, align 8 +; IR-ITERATIVE-NEXT: br label [[TMP16]] +; IR-ITERATIVE: 16: +; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ] +; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = bitcast double [[TMP17]] to i64 +; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = trunc i64 [[TMP18]] to i32 +; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = lshr i64 [[TMP18]], 32 +; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = trunc i64 [[TMP20]] to i32 +; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP19]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP23:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP21]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = insertelement <2 x i32> poison, i32 [[TMP22]], i32 0 +; IR-ITERATIVE-NEXT: [[TMP25:%.*]] = insertelement <2 x i32> [[TMP24]], i32 [[TMP23]], i32 1 +; IR-ITERATIVE-NEXT: [[TMP26:%.*]] = bitcast <2 x i32> [[TMP25]] to double +; IR-ITERATIVE-NEXT: [[TMP27:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP28:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP27]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP29:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP26]], double [[TMP28]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: br label [[TMP30]] +; IR-ITERATIVE: 30: +; IR-ITERATIVE-NEXT: [[TMP31:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP29]], [[TMP16]] ] +; IR-ITERATIVE-NEXT: ret double [[TMP31]] +; +; IR-DPP-LABEL: @global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp( +; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]] +; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP30:%.*]] +; IR-DPP: 2: +; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-DPP-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR-DPP: 14: +; IR-DPP-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("one-as") monotonic, align 8 +; IR-DPP-NEXT: br label [[TMP16]] +; IR-DPP: 16: +; IR-DPP-NEXT: [[TMP17:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ] +; IR-DPP-NEXT: [[TMP18:%.*]] = bitcast double [[TMP17]] to i64 +; IR-DPP-NEXT: [[TMP19:%.*]] = trunc i64 [[TMP18]] to i32 +; IR-DPP-NEXT: [[TMP20:%.*]] = lshr i64 [[TMP18]], 32 +; IR-DPP-NEXT: [[TMP21:%.*]] = trunc i64 [[TMP20]] to i32 +; IR-DPP-NEXT: [[TMP22:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP19]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP23:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP21]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP24:%.*]] = insertelement <2 x i32> poison, i32 [[TMP22]], i32 0 +; IR-DPP-NEXT: [[TMP25:%.*]] = insertelement <2 x i32> [[TMP24]], i32 [[TMP23]], i32 1 +; IR-DPP-NEXT: [[TMP26:%.*]] = bitcast <2 x i32> [[TMP25]] to double +; IR-DPP-NEXT: [[TMP27:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP28:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP27]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP29:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP26]], double [[TMP28]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: br label [[TMP30]] +; IR-DPP: 30: +; IR-DPP-NEXT: [[TMP31:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP29]], [[TMP16]] ] +; IR-DPP-NEXT: ret double [[TMP31]] +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("one-as") monotonic + ret double %result +} + +define amdgpu_ps double @global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_structfp(ptr addrspace(1) inreg %ptr, double %val) #1 { +; IR-LABEL: @global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_structfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("one-as") monotonic, align 8 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("one-as") monotonic + ret double %result +} + +define amdgpu_ps double @global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp(ptr addrspace(1) inreg %ptr, double inreg %val) #2 { +; IR-ITERATIVE-LABEL: @global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp( +; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]] +; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP30:%.*]] +; IR-ITERATIVE: 2: +; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-ITERATIVE-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR-ITERATIVE: 14: +; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("agent") monotonic, align 8 +; IR-ITERATIVE-NEXT: br label [[TMP16]] +; IR-ITERATIVE: 16: +; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ] +; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = bitcast double [[TMP17]] to i64 +; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = trunc i64 [[TMP18]] to i32 +; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = lshr i64 [[TMP18]], 32 +; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = trunc i64 [[TMP20]] to i32 +; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP19]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP23:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP21]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = insertelement <2 x i32> poison, i32 [[TMP22]], i32 0 +; IR-ITERATIVE-NEXT: [[TMP25:%.*]] = insertelement <2 x i32> [[TMP24]], i32 [[TMP23]], i32 1 +; IR-ITERATIVE-NEXT: [[TMP26:%.*]] = bitcast <2 x i32> [[TMP25]] to double +; IR-ITERATIVE-NEXT: [[TMP27:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP28:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP27]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP29:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP26]], double [[TMP28]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: br label [[TMP30]] +; IR-ITERATIVE: 30: +; IR-ITERATIVE-NEXT: [[TMP31:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP29]], [[TMP16]] ] +; IR-ITERATIVE-NEXT: ret double [[TMP31]] +; +; IR-DPP-LABEL: @global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp( +; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]] +; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP30:%.*]] +; IR-DPP: 2: +; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-DPP-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR-DPP: 14: +; IR-DPP-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("agent") monotonic, align 8 +; IR-DPP-NEXT: br label [[TMP16]] +; IR-DPP: 16: +; IR-DPP-NEXT: [[TMP17:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ] +; IR-DPP-NEXT: [[TMP18:%.*]] = bitcast double [[TMP17]] to i64 +; IR-DPP-NEXT: [[TMP19:%.*]] = trunc i64 [[TMP18]] to i32 +; IR-DPP-NEXT: [[TMP20:%.*]] = lshr i64 [[TMP18]], 32 +; IR-DPP-NEXT: [[TMP21:%.*]] = trunc i64 [[TMP20]] to i32 +; IR-DPP-NEXT: [[TMP22:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP19]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP23:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP21]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP24:%.*]] = insertelement <2 x i32> poison, i32 [[TMP22]], i32 0 +; IR-DPP-NEXT: [[TMP25:%.*]] = insertelement <2 x i32> [[TMP24]], i32 [[TMP23]], i32 1 +; IR-DPP-NEXT: [[TMP26:%.*]] = bitcast <2 x i32> [[TMP25]] to double +; IR-DPP-NEXT: [[TMP27:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP28:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP27]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP29:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP26]], double [[TMP28]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: br label [[TMP30]] +; IR-DPP: 30: +; IR-DPP-NEXT: [[TMP31:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP29]], [[TMP16]] ] +; IR-DPP-NEXT: ret double [[TMP31]] +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret double %result +} + +define amdgpu_ps double @global_atomic_fsub_double_uni_address_div_value_agent_scope_strictfp(ptr addrspace(1) inreg %ptr, double %val) #2 { +; IR-LABEL: @global_atomic_fsub_double_uni_address_div_value_agent_scope_strictfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fsub ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fsub ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret double %result +} + +define amdgpu_ps double @global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) inreg %ptr, double inreg %val) #0 { +; IR-LABEL: @global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe( +; IR-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() +; IR-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP26:%.*]] +; IR: 2: +; IR-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) +; IR-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) +; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) +; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]] +; IR: 10: +; IR-NEXT: [[TMP11:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: br label [[TMP12]] +; IR: 12: +; IR-NEXT: [[TMP13:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP11]], [[TMP10]] ] +; IR-NEXT: [[TMP14:%.*]] = bitcast double [[TMP13]] to i64 +; IR-NEXT: [[TMP15:%.*]] = trunc i64 [[TMP14]] to i32 +; IR-NEXT: [[TMP16:%.*]] = lshr i64 [[TMP14]], 32 +; IR-NEXT: [[TMP17:%.*]] = trunc i64 [[TMP16]] to i32 +; IR-NEXT: [[TMP18:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP15]]) +; IR-NEXT: [[TMP19:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP17]]) +; IR-NEXT: [[TMP20:%.*]] = insertelement <2 x i32> poison, i32 [[TMP18]], i32 0 +; IR-NEXT: [[TMP21:%.*]] = insertelement <2 x i32> [[TMP20]], i32 [[TMP19]], i32 1 +; IR-NEXT: [[TMP22:%.*]] = bitcast <2 x i32> [[TMP21]] to double +; IR-NEXT: [[TMP23:%.*]] = uitofp i32 [[TMP8]] to double +; IR-NEXT: [[TMP24:%.*]] = select i1 [[TMP9]], double 0x7FF0000000000000, double [[VAL]] +; IR-NEXT: [[TMP25:%.*]] = call double @llvm.minnum.f64(double [[TMP22]], double [[TMP24]]) +; IR-NEXT: br label [[TMP26]] +; IR: 26: +; IR-NEXT: [[TMP27:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP25]], [[TMP12]] ] +; IR-NEXT: ret double [[TMP27]] +; + %result = atomicrmw fmin ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret double %result +} + +define amdgpu_ps double @global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe(ptr addrspace(1) inreg %ptr, double %val) #0 { +; IR-LABEL: @global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fmin ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret double %result +} + +define amdgpu_ps double @global_atomic__fmax_double_uni_address_uni_value_agent_scope_unsafe_structfp(ptr addrspace(1) inreg %ptr, double inreg %val) #1{ +; IR-ITERATIVE-LABEL: @global_atomic__fmax_double_uni_address_uni_value_agent_scope_unsafe_structfp( +; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]] +; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP26:%.*]] +; IR-ITERATIVE: 2: +; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-ITERATIVE-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]] +; IR-ITERATIVE: 10: +; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-ITERATIVE-NEXT: br label [[TMP12]] +; IR-ITERATIVE: 12: +; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP11]], [[TMP10]] ] +; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = bitcast double [[TMP13]] to i64 +; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = trunc i64 [[TMP14]] to i32 +; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = lshr i64 [[TMP14]], 32 +; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = trunc i64 [[TMP16]] to i32 +; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP15]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP17]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = insertelement <2 x i32> poison, i32 [[TMP18]], i32 0 +; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = insertelement <2 x i32> [[TMP20]], i32 [[TMP19]], i32 1 +; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = bitcast <2 x i32> [[TMP21]] to double +; IR-ITERATIVE-NEXT: [[TMP23:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = select i1 [[TMP9]], double 0xFFF0000000000000, double [[VAL]] +; IR-ITERATIVE-NEXT: [[TMP25:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP22]], double [[TMP24]], metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: br label [[TMP26]] +; IR-ITERATIVE: 26: +; IR-ITERATIVE-NEXT: [[TMP27:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP25]], [[TMP12]] ] +; IR-ITERATIVE-NEXT: ret double [[TMP27]] +; +; IR-DPP-LABEL: @global_atomic__fmax_double_uni_address_uni_value_agent_scope_unsafe_structfp( +; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]] +; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP26:%.*]] +; IR-DPP: 2: +; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-DPP-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]] +; IR-DPP: 10: +; IR-DPP-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-DPP-NEXT: br label [[TMP12]] +; IR-DPP: 12: +; IR-DPP-NEXT: [[TMP13:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP11]], [[TMP10]] ] +; IR-DPP-NEXT: [[TMP14:%.*]] = bitcast double [[TMP13]] to i64 +; IR-DPP-NEXT: [[TMP15:%.*]] = trunc i64 [[TMP14]] to i32 +; IR-DPP-NEXT: [[TMP16:%.*]] = lshr i64 [[TMP14]], 32 +; IR-DPP-NEXT: [[TMP17:%.*]] = trunc i64 [[TMP16]] to i32 +; IR-DPP-NEXT: [[TMP18:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP15]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP19:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP17]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP20:%.*]] = insertelement <2 x i32> poison, i32 [[TMP18]], i32 0 +; IR-DPP-NEXT: [[TMP21:%.*]] = insertelement <2 x i32> [[TMP20]], i32 [[TMP19]], i32 1 +; IR-DPP-NEXT: [[TMP22:%.*]] = bitcast <2 x i32> [[TMP21]] to double +; IR-DPP-NEXT: [[TMP23:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP24:%.*]] = select i1 [[TMP9]], double 0xFFF0000000000000, double [[VAL]] +; IR-DPP-NEXT: [[TMP25:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP22]], double [[TMP24]], metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: br label [[TMP26]] +; IR-DPP: 26: +; IR-DPP-NEXT: [[TMP27:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP25]], [[TMP12]] ] +; IR-DPP-NEXT: ret double [[TMP27]] +; + %result = atomicrmw fmax ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret double %result +} + +define amdgpu_ps double @global_atomic__fmax_double_uni_address_div_value_agent_scope_unsafe_structfp(ptr addrspace(1) inreg %ptr, double %val) #1{ +; IR-LABEL: @global_atomic__fmax_double_uni_address_div_value_agent_scope_unsafe_structfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fmax ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret double %result +} + +define amdgpu_ps double @global_atomic_fadd_double_uni_address_uni_value_system_scope_strictfp(ptr addrspace(1) inreg %ptr, double inreg %val) #2 { +; IR-ITERATIVE-LABEL: @global_atomic_fadd_double_uni_address_uni_value_system_scope_strictfp( +; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]] +; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP30:%.*]] +; IR-ITERATIVE: 2: +; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-ITERATIVE-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR-ITERATIVE: 14: +; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] monotonic, align 4 +; IR-ITERATIVE-NEXT: br label [[TMP16]] +; IR-ITERATIVE: 16: +; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ] +; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = bitcast double [[TMP17]] to i64 +; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = trunc i64 [[TMP18]] to i32 +; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = lshr i64 [[TMP18]], 32 +; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = trunc i64 [[TMP20]] to i32 +; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP19]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP23:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP21]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = insertelement <2 x i32> poison, i32 [[TMP22]], i32 0 +; IR-ITERATIVE-NEXT: [[TMP25:%.*]] = insertelement <2 x i32> [[TMP24]], i32 [[TMP23]], i32 1 +; IR-ITERATIVE-NEXT: [[TMP26:%.*]] = bitcast <2 x i32> [[TMP25]] to double +; IR-ITERATIVE-NEXT: [[TMP27:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP28:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP27]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP29:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP26]], double [[TMP28]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: br label [[TMP30]] +; IR-ITERATIVE: 30: +; IR-ITERATIVE-NEXT: [[TMP31:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP29]], [[TMP16]] ] +; IR-ITERATIVE-NEXT: ret double [[TMP31]] +; +; IR-DPP-LABEL: @global_atomic_fadd_double_uni_address_uni_value_system_scope_strictfp( +; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]] +; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP30:%.*]] +; IR-DPP: 2: +; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-DPP-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR-DPP: 14: +; IR-DPP-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] monotonic, align 4 +; IR-DPP-NEXT: br label [[TMP16]] +; IR-DPP: 16: +; IR-DPP-NEXT: [[TMP17:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ] +; IR-DPP-NEXT: [[TMP18:%.*]] = bitcast double [[TMP17]] to i64 +; IR-DPP-NEXT: [[TMP19:%.*]] = trunc i64 [[TMP18]] to i32 +; IR-DPP-NEXT: [[TMP20:%.*]] = lshr i64 [[TMP18]], 32 +; IR-DPP-NEXT: [[TMP21:%.*]] = trunc i64 [[TMP20]] to i32 +; IR-DPP-NEXT: [[TMP22:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP19]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP23:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP21]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP24:%.*]] = insertelement <2 x i32> poison, i32 [[TMP22]], i32 0 +; IR-DPP-NEXT: [[TMP25:%.*]] = insertelement <2 x i32> [[TMP24]], i32 [[TMP23]], i32 1 +; IR-DPP-NEXT: [[TMP26:%.*]] = bitcast <2 x i32> [[TMP25]] to double +; IR-DPP-NEXT: [[TMP27:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP28:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP27]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP29:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP26]], double [[TMP28]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: br label [[TMP30]] +; IR-DPP: 30: +; IR-DPP-NEXT: [[TMP31:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP29]], [[TMP16]] ] +; IR-DPP-NEXT: ret double [[TMP31]] +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val monotonic, align 4 + ret double %result +} + +define amdgpu_ps double @global_atomic_fadd_double_uni_address_div_value_system_scope_strictfp(ptr addrspace(1) inreg %ptr, double %val) #2 { +; IR-LABEL: @global_atomic_fadd_double_uni_address_div_value_system_scope_strictfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] monotonic, align 4 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val monotonic, align 4 + ret double %result +} + +define amdgpu_ps double @global_atomic_fadd_double_div_address_uni_value_agent_scope_unsafe(ptr addrspace(1) %ptr, double inreg %val) #0 { +; IR-LABEL: @global_atomic_fadd_double_div_address_uni_value_agent_scope_unsafe( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 4 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic, align 4 + ret double %result +} + +define amdgpu_ps double @global_atomic_fadd_double_div_address_div_value_agent_scope_unsafe(ptr addrspace(1) %ptr, double %val) #0 { +; IR-LABEL: @global_atomic_fadd_double_div_address_div_value_agent_scope_unsafe( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 4 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic, align 4 + ret double %result +} + +define amdgpu_ps double @global_atomic_fadd_double_div_address_uni_value_one_as_scope_unsafe_structfp(ptr addrspace(1) %ptr, double inreg %val) #1 { +; IR-LABEL: @global_atomic_fadd_double_div_address_uni_value_one_as_scope_unsafe_structfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("one-as") monotonic, align 8 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("one-as") monotonic + ret double %result +} + +define amdgpu_ps double @global_atomic_fadd_double_div_address_div_value_one_as_scope_unsafe_structfp(ptr addrspace(1) %ptr, double %val) #1 { +; IR-LABEL: @global_atomic_fadd_double_div_address_div_value_one_as_scope_unsafe_structfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("one-as") monotonic, align 8 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("one-as") monotonic + ret double %result +} + +define amdgpu_ps double @global_atomic_fsub_double_div_address_uni_value_agent_scope_strictfp(ptr addrspace(1) %ptr, double inreg %val) #2 { +; IR-LABEL: @global_atomic_fsub_double_div_address_uni_value_agent_scope_strictfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret double %result +} + +define amdgpu_ps double @global_atomic_fsub_double_div_address_div_value_agent_scope_strictfp(ptr addrspace(1) %ptr, double %val) #2 { +; IR-LABEL: @global_atomic_fsub_double_div_address_div_value_agent_scope_strictfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fsub ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fsub ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret double %result +} + +define amdgpu_ps double @global_atomic_fmin_double_div_address_uni_value_agent_scope(ptr addrspace(1) %ptr, double inreg %val) #0 { +; IR-LABEL: @global_atomic_fmin_double_div_address_uni_value_agent_scope( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fmin ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret double %result +} + +define amdgpu_ps double @global_atomic_fmin_double_div_address_div_value_agent_scope(ptr addrspace(1) %ptr, double %val) #0 { +; IR-LABEL: @global_atomic_fmin_double_div_address_div_value_agent_scope( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fmin ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret double %result +} + +define amdgpu_ps double @global_atomic__fmax_double_div_address_uni_value_agent_scope_unsafe_structfp(ptr addrspace(1) %ptr, double inreg %val) #1{ +; IR-LABEL: @global_atomic__fmax_double_div_address_uni_value_agent_scope_unsafe_structfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fmax ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret double %result +} + +define amdgpu_ps double @global_atomic__fmax_double_div_address_div_value_agent_scope_unsafe_structfp(ptr addrspace(1) %ptr, double %val) #1{ +; IR-LABEL: @global_atomic__fmax_double_div_address_div_value_agent_scope_unsafe_structfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fmax ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret double %result +} + +define amdgpu_ps double @global_atomic_fadd_double_div_address_uni_value_system_scope_strictfp(ptr addrspace(1) %ptr, double inreg %val) #2 { +; IR-LABEL: @global_atomic_fadd_double_div_address_uni_value_system_scope_strictfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] monotonic, align 4 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val monotonic, align 4 + ret double %result +} + +define amdgpu_ps double @global_atomic_fadd_double_div_address_div_value_system_scope_strictfp(ptr addrspace(1) %ptr, double %val) #2 { +; IR-LABEL: @global_atomic_fadd_double_div_address_div_value_system_scope_strictfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] monotonic, align 4 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val monotonic, align 4 + ret double %result +} + attributes #0 = { "denormal-fp-math-f32"="preserve-sign,preserve-sign" "amdgpu-unsafe-fp-atomics"="true" } attributes #1 = { strictfp "denormal-fp-math-f32"="preserve-sign,preserve-sign" "amdgpu-unsafe-fp-atomics"="true" } attributes #2 = { strictfp } diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_optimizer_fp_no_rtn.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_optimizer_fp_no_rtn.ll index e70d7347890f..b9234f47df19 100644 --- a/llvm/test/CodeGen/AMDGPU/global_atomics_optimizer_fp_no_rtn.ll +++ b/llvm/test/CodeGen/AMDGPU/global_atomics_optimizer_fp_no_rtn.ll @@ -864,6 +864,426 @@ define amdgpu_ps void @global_atomic_fadd_div_address_div_value_system_scope_str ret void } +define amdgpu_ps void @global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) inreg %ptr, double inreg %val) #0 { +; IR-LABEL: @global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe( +; IR-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() +; IR-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]] +; IR: 2: +; IR-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) +; IR-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) +; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) +; IR-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) +; IR-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-NEXT: [[TMP11:%.*]] = uitofp i32 [[TMP10]] to double +; IR-NEXT: [[TMP12:%.*]] = fmul double [[VAL:%.*]], [[TMP11]] +; IR-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR: 14: +; IR-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("agent") monotonic, align 4 +; IR-NEXT: br label [[TMP16]] +; IR: 16: +; IR-NEXT: br label [[TMP17]] +; IR: 17: +; IR-NEXT: ret void +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic, align 4 + ret void +} + +define amdgpu_ps void @global_atomic_fadd_double_uni_address_div_value_scope_agent_scope_unsafe(ptr addrspace(1) inreg %ptr, double %val) #0 { +; IR-LABEL: @global_atomic_fadd_double_uni_address_div_value_scope_agent_scope_unsafe( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 4 +; IR-NEXT: ret void +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic, align 4 + ret void +} + +define amdgpu_ps void @global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp(ptr addrspace(1) inreg %ptr, double inreg %val) #1 { +; IR-ITERATIVE-LABEL: @global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp( +; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]] +; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]] +; IR-ITERATIVE: 2: +; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-ITERATIVE-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR-ITERATIVE: 14: +; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("one-as") monotonic, align 8 +; IR-ITERATIVE-NEXT: br label [[TMP16]] +; IR-ITERATIVE: 16: +; IR-ITERATIVE-NEXT: br label [[TMP17]] +; IR-ITERATIVE: 17: +; IR-ITERATIVE-NEXT: ret void +; +; IR-DPP-LABEL: @global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp( +; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]] +; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]] +; IR-DPP: 2: +; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-DPP-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR-DPP: 14: +; IR-DPP-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("one-as") monotonic, align 8 +; IR-DPP-NEXT: br label [[TMP16]] +; IR-DPP: 16: +; IR-DPP-NEXT: br label [[TMP17]] +; IR-DPP: 17: +; IR-DPP-NEXT: ret void +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("one-as") monotonic + ret void +} + +define amdgpu_ps void @global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_structfp(ptr addrspace(1) inreg %ptr, double %val) #1 { +; IR-LABEL: @global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_structfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("one-as") monotonic, align 8 +; IR-NEXT: ret void +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("one-as") monotonic + ret void +} + +define amdgpu_ps void @global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp(ptr addrspace(1) inreg %ptr, double inreg %val) #2 { +; IR-ITERATIVE-LABEL: @global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp( +; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]] +; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]] +; IR-ITERATIVE: 2: +; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-ITERATIVE-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR-ITERATIVE: 14: +; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("agent") monotonic, align 8 +; IR-ITERATIVE-NEXT: br label [[TMP16]] +; IR-ITERATIVE: 16: +; IR-ITERATIVE-NEXT: br label [[TMP17]] +; IR-ITERATIVE: 17: +; IR-ITERATIVE-NEXT: ret void +; +; IR-DPP-LABEL: @global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp( +; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]] +; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]] +; IR-DPP: 2: +; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-DPP-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR-DPP: 14: +; IR-DPP-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("agent") monotonic, align 8 +; IR-DPP-NEXT: br label [[TMP16]] +; IR-DPP: 16: +; IR-DPP-NEXT: br label [[TMP17]] +; IR-DPP: 17: +; IR-DPP-NEXT: ret void +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret void +} + +define amdgpu_ps void @global_atomic_fsub_double_uni_address_div_value_agent_scope_strictfp(ptr addrspace(1) inreg %ptr, double %val) #2 { +; IR-LABEL: @global_atomic_fsub_double_uni_address_div_value_agent_scope_strictfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fsub ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret void +; + %result = atomicrmw fsub ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret void +} + +define amdgpu_ps void @global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) inreg %ptr, double inreg %val) #0 { +; IR-LABEL: @global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe( +; IR-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() +; IR-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP13:%.*]] +; IR: 2: +; IR-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) +; IR-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) +; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) +; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]] +; IR: 10: +; IR-NEXT: [[TMP11:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: br label [[TMP12]] +; IR: 12: +; IR-NEXT: br label [[TMP13]] +; IR: 13: +; IR-NEXT: ret void +; + %result = atomicrmw fmin ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret void +} + +define amdgpu_ps void @global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe(ptr addrspace(1) inreg %ptr, double %val) #0 { +; IR-LABEL: @global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret void +; + %result = atomicrmw fmin ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret void +} + +define amdgpu_ps void @global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe_structfp(ptr addrspace(1) inreg %ptr, double inreg %val) #1{ +; IR-ITERATIVE-LABEL: @global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe_structfp( +; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]] +; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP13:%.*]] +; IR-ITERATIVE: 2: +; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-ITERATIVE-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]] +; IR-ITERATIVE: 10: +; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-ITERATIVE-NEXT: br label [[TMP12]] +; IR-ITERATIVE: 12: +; IR-ITERATIVE-NEXT: br label [[TMP13]] +; IR-ITERATIVE: 13: +; IR-ITERATIVE-NEXT: ret void +; +; IR-DPP-LABEL: @global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe_structfp( +; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]] +; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP13:%.*]] +; IR-DPP: 2: +; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-DPP-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]] +; IR-DPP: 10: +; IR-DPP-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-DPP-NEXT: br label [[TMP12]] +; IR-DPP: 12: +; IR-DPP-NEXT: br label [[TMP13]] +; IR-DPP: 13: +; IR-DPP-NEXT: ret void +; + %result = atomicrmw fmax ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret void +} + +define amdgpu_ps void @global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe_structfp(ptr addrspace(1) inreg %ptr, double %val) #1{ +; IR-LABEL: @global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe_structfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret void +; + %result = atomicrmw fmax ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret void +} + +define amdgpu_ps void @global_atomic_fadd_double_uni_address_uni_value_system_scope_strictfp(ptr addrspace(1) inreg %ptr, double inreg %val) #2 { +; IR-ITERATIVE-LABEL: @global_atomic_fadd_double_uni_address_uni_value_system_scope_strictfp( +; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]] +; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]] +; IR-ITERATIVE: 2: +; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-ITERATIVE-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR-ITERATIVE: 14: +; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] monotonic, align 4 +; IR-ITERATIVE-NEXT: br label [[TMP16]] +; IR-ITERATIVE: 16: +; IR-ITERATIVE-NEXT: br label [[TMP17]] +; IR-ITERATIVE: 17: +; IR-ITERATIVE-NEXT: ret void +; +; IR-DPP-LABEL: @global_atomic_fadd_double_uni_address_uni_value_system_scope_strictfp( +; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]] +; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]] +; IR-DPP: 2: +; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-DPP-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR-DPP: 14: +; IR-DPP-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] monotonic, align 4 +; IR-DPP-NEXT: br label [[TMP16]] +; IR-DPP: 16: +; IR-DPP-NEXT: br label [[TMP17]] +; IR-DPP: 17: +; IR-DPP-NEXT: ret void +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val monotonic, align 4 + ret void +} + +define amdgpu_ps void @global_atomic_fadd_double_uni_address_div_value_system_scope_strictfp(ptr addrspace(1) inreg %ptr, double %val) #2 { +; IR-LABEL: @global_atomic_fadd_double_uni_address_div_value_system_scope_strictfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] monotonic, align 4 +; IR-NEXT: ret void +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val monotonic, align 4 + ret void +} + +define amdgpu_ps void @global_atomic_fadd_double_div_address_uni_value_agent_scope_unsafe(ptr addrspace(1) %ptr, double inreg %val) #0 { +; IR-LABEL: @global_atomic_fadd_double_div_address_uni_value_agent_scope_unsafe( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 4 +; IR-NEXT: ret void +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic, align 4 + ret void +} + +define amdgpu_ps void @global_atomic_fadd_double_div_address_div_value_agent_scope_unsafe(ptr addrspace(1) %ptr, double %val) #0 { +; IR-LABEL: @global_atomic_fadd_double_div_address_div_value_agent_scope_unsafe( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 4 +; IR-NEXT: ret void +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic, align 4 + ret void +} + +define amdgpu_ps void @global_atomic_fadd_double_div_address_uni_value_one_as_scope_unsafe_structfp(ptr addrspace(1) %ptr, double inreg %val) #1 { +; IR-LABEL: @global_atomic_fadd_double_div_address_uni_value_one_as_scope_unsafe_structfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("one-as") monotonic, align 8 +; IR-NEXT: ret void +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("one-as") monotonic + ret void +} + +define amdgpu_ps void @global_atomic_fadd_double_div_address_div_value_one_as_scope_unsafe_structfp(ptr addrspace(1) %ptr, double %val) #1 { +; IR-LABEL: @global_atomic_fadd_double_div_address_div_value_one_as_scope_unsafe_structfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("one-as") monotonic, align 8 +; IR-NEXT: ret void +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("one-as") monotonic + ret void +} + +define amdgpu_ps void @global_atomic_fsub_double_div_address_uni_value_agent_scope_strictfp(ptr addrspace(1) %ptr, double inreg %val) #2 { +; IR-LABEL: @global_atomic_fsub_double_div_address_uni_value_agent_scope_strictfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret void +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret void +} + +define amdgpu_ps void @global_atomic_fsub_double_div_address_div_value_agent_scope_strictfp(ptr addrspace(1) %ptr, double %val) #2 { +; IR-LABEL: @global_atomic_fsub_double_div_address_div_value_agent_scope_strictfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fsub ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret void +; + %result = atomicrmw fsub ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret void +} + +define amdgpu_ps void @global_atomic_fmin_double_div_address_uni_value_agent_scope(ptr addrspace(1) %ptr, double inreg %val) #0 { +; IR-LABEL: @global_atomic_fmin_double_div_address_uni_value_agent_scope( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret void +; + %result = atomicrmw fmin ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret void +} + +define amdgpu_ps void @global_atomic_fmin_double_div_address_div_value_agent_scope(ptr addrspace(1) %ptr, double %val) #0 { +; IR-LABEL: @global_atomic_fmin_double_div_address_div_value_agent_scope( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret void +; + %result = atomicrmw fmin ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret void +} + +define amdgpu_ps void @global_atomic_fmax_double_div_address_uni_value_agent_scope_unsafe_structfp(ptr addrspace(1) %ptr, double inreg %val) #1{ +; IR-LABEL: @global_atomic_fmax_double_div_address_uni_value_agent_scope_unsafe_structfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret void +; + %result = atomicrmw fmax ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret void +} + +define amdgpu_ps void @global_atomic_fmax_double_div_address_div_value_agent_scope_unsafe_structfp(ptr addrspace(1) %ptr, double %val) #1{ +; IR-LABEL: @global_atomic_fmax_double_div_address_div_value_agent_scope_unsafe_structfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret void +; + %result = atomicrmw fmax ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret void +} + +define amdgpu_ps void @global_atomic_fadd_double_div_address_uni_value_system_scope_strictfp(ptr addrspace(1) %ptr, double inreg %val) #2 { +; IR-LABEL: @global_atomic_fadd_double_div_address_uni_value_system_scope_strictfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] monotonic, align 4 +; IR-NEXT: ret void +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val monotonic, align 4 + ret void +} + +define amdgpu_ps void @global_atomic_fadd_double_div_address_div_value_system_scope_strictfp(ptr addrspace(1) %ptr, double %val) #2 { +; IR-LABEL: @global_atomic_fadd_double_div_address_div_value_system_scope_strictfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] monotonic, align 4 +; IR-NEXT: ret void +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val monotonic, align 4 + ret void +} + attributes #0 = { "denormal-fp-math-f32"="preserve-sign,preserve-sign" "amdgpu-unsafe-fp-atomics"="true" } attributes #1 = { strictfp "denormal-fp-math-f32"="preserve-sign,preserve-sign" "amdgpu-unsafe-fp-atomics"="true" } attributes #2 = { strictfp } diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll index 96c615b974ce..4f00d48551be 100644 --- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll +++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll @@ -13,6 +13,7 @@ ; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,-wavefrontsize64 -amdgpu-atomic-optimizer-strategy=DPP -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX1132-DPP %s declare float @div.float.value() +declare double @div.double.value() define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 { ; GFX7LESS-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe: @@ -5408,6 +5409,5583 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_defalut_scop ret void } +define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_movk_i32 s32, 0x800 +; GFX7LESS-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s42, -1 +; GFX7LESS-NEXT: s_mov_b32 s43, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s40, s40, s3 +; GFX7LESS-NEXT: s_addc_u32 s41, s41, 0 +; GFX7LESS-NEXT: s_mov_b32 s33, s2 +; GFX7LESS-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX7LESS-NEXT: v_mov_b32_e32 v40, v0 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec +; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s0, 0 +; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s1, v0 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX7LESS-NEXT: s_cbranch_execz .LBB9_3 +; GFX7LESS-NEXT: ; %bb.1: +; GFX7LESS-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x9 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_load_dwordx2 s[2:3], s[36:37], 0x0 +; GFX7LESS-NEXT: s_bcnt1_i32_b64 s0, s[0:1] +; GFX7LESS-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX7LESS-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX7LESS-NEXT: s_mov_b64 s[38:39], 0 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, s2 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, s3 +; GFX7LESS-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_add_f64 v[2:3], v[0:1], v[41:42] +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[40:43], 0 offset:4 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[40:43], 0 +; GFX7LESS-NEXT: s_add_u32 s8, s34, 44 +; GFX7LESS-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:12 +; GFX7LESS-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:8 +; GFX7LESS-NEXT: s_addc_u32 s9, s35, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX7LESS-NEXT: s_waitcnt expcnt(2) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, 0 +; GFX7LESS-NEXT: s_mov_b32 s12, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v40 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s36 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s37 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX7LESS-NEXT: v_and_b32_e32 v2, 1, v0 +; GFX7LESS-NEXT: buffer_load_dword v0, off, s[40:43], 0 +; GFX7LESS-NEXT: buffer_load_dword v1, off, s[40:43], 0 offset:4 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2 +; GFX7LESS-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB9_2 +; GFX7LESS-NEXT: .LBB9_3: +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s42, -1 +; GFX9-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX9-NEXT: s_mov_b64 s[0:1], exec +; GFX9-NEXT: s_mov_b32 s43, 0xe00000 +; GFX9-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0 +; GFX9-NEXT: s_add_u32 s40, s40, s3 +; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0 +; GFX9-NEXT: s_addc_u32 s41, s41, 0 +; GFX9-NEXT: s_mov_b32 s33, s2 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-NEXT: s_movk_i32 s32, 0x800 +; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX9-NEXT: s_cbranch_execz .LBB9_3 +; GFX9-NEXT: ; %bb.1: +; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1] +; GFX9-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX9-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX9-NEXT: s_mov_b64 s[38:39], 0 +; GFX9-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX9-NEXT: s_add_u32 s8, s34, 44 +; GFX9-NEXT: s_addc_u32 s9, s35, 0 +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX9-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX9-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX9-NEXT: s_mov_b32 s12, s33 +; GFX9-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX9-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX9-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX9-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: v_mov_b32_e32 v2, s36 +; GFX9-NEXT: v_mov_b32_e32 v3, s37 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX9-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX9-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX9-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX9-NEXT: s_cbranch_execnz .LBB9_2 +; GFX9-NEXT: .LBB9_3: +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s42, -1 +; GFX1064-NEXT: s_mov_b32 s43, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s40, s40, s3 +; GFX1064-NEXT: s_mov_b32 s33, s2 +; GFX1064-NEXT: s_mov_b64 s[2:3], exec +; GFX1064-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1064-NEXT: s_addc_u32 s41, s41, 0 +; GFX1064-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1064-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX1064-NEXT: s_cbranch_execz .LBB9_3 +; GFX1064-NEXT: ; %bb.1: +; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[2:3] +; GFX1064-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1064-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX1064-NEXT: s_mov_b64 s[38:39], 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1064-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: v_mov_b32_e32 v2, s1 +; GFX1064-NEXT: v_mov_b32_e32 v1, s0 +; GFX1064-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1064-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1064-NEXT: s_mov_b32 s12, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1064-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1064-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1064-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1064-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-NEXT: v_mov_b32_e32 v2, s36 +; GFX1064-NEXT: v_mov_b32_e32 v3, s37 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1064-NEXT: s_clause 0x1 +; GFX1064-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1064-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX1064-NEXT: s_cbranch_execnz .LBB9_2 +; GFX1064-NEXT: .LBB9_3: +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s33, s2 +; GFX1032-NEXT: s_mov_b32 s2, exec_lo +; GFX1032-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1032-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s42, -1 +; GFX1032-NEXT: s_mov_b32 s43, 0x31c16000 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-NEXT: s_add_u32 s40, s40, s3 +; GFX1032-NEXT: s_addc_u32 s41, s41, 0 +; GFX1032-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1032-NEXT: s_mov_b32 s38, 0 +; GFX1032-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo +; GFX1032-NEXT: s_cbranch_execz .LBB9_3 +; GFX1032-NEXT: ; %bb.1: +; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s2 +; GFX1032-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1032-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1032-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: v_mov_b32_e32 v2, s1 +; GFX1032-NEXT: v_mov_b32_e32 v1, s0 +; GFX1032-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1032-NEXT: s_mov_b32 s12, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1032-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1032-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1032-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1032-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-NEXT: v_mov_b32_e32 v2, s36 +; GFX1032-NEXT: v_mov_b32_e32 v3, s37 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1032-NEXT: s_clause 0x1 +; GFX1032-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1032-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s38 +; GFX1032-NEXT: s_cbranch_execnz .LBB9_2 +; GFX1032-NEXT: .LBB9_3: +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b32 s33, s2 +; GFX1164-NEXT: s_mov_b64 s[2:3], exec +; GFX1164-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1164-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1164-NEXT: s_mov_b32 s32, 32 +; GFX1164-NEXT: s_mov_b64 s[0:1], exec +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-NEXT: s_cbranch_execz .LBB9_3 +; GFX1164-NEXT: ; %bb.1: +; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[2:3] +; GFX1164-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1164-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX1164-NEXT: s_mov_b64 s[38:39], 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: v_mov_b32_e32 v2, s1 +; GFX1164-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-NEXT: .p2align 6 +; GFX1164-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-NEXT: s_mov_b32 s12, s33 +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-NEXT: v_mov_b32_e32 v2, s36 +; GFX1164-NEXT: v_mov_b32_e32 v3, s37 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[38:39] +; GFX1164-NEXT: s_cbranch_execnz .LBB9_2 +; GFX1164-NEXT: .LBB9_3: +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_mov_b32 s2, exec_lo +; GFX1132-NEXT: v_mov_b32_e32 v40, v0 +; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1132-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1132-NEXT: s_mov_b32 s38, 0 +; GFX1132-NEXT: s_mov_b32 s32, 32 +; GFX1132-NEXT: s_mov_b32 s0, exec_lo +; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-NEXT: s_cbranch_execz .LBB9_3 +; GFX1132-NEXT: ; %bb.1: +; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s2 +; GFX1132-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1132-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX1132-NEXT: s_mov_b32 s33, s15 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0 +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-NEXT: .p2align 6 +; GFX1132-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-NEXT: v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-NEXT: s_mov_b32 s12, s33 +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s36 +; GFX1132-NEXT: v_dual_mov_b32 v3, s37 :: v_dual_mov_b32 v4, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38 +; GFX1132-NEXT: s_cbranch_execnz .LBB9_2 +; GFX1132-NEXT: .LBB9_3: +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s42, -1 +; GFX9-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], exec +; GFX9-DPP-NEXT: s_mov_b32 s43, 0xe00000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0 +; GFX9-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0 +; GFX9-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX9-DPP-NEXT: s_mov_b32 s33, s2 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX9-DPP-NEXT: s_cbranch_execz .LBB9_3 +; GFX9-DPP-NEXT: ; %bb.1: +; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1] +; GFX9-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX9-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX9-DPP-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-DPP-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX9-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX9-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX9-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX9-DPP-NEXT: s_mov_b32 s12, s33 +; GFX9-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX9-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX9-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX9-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX9-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB9_2 +; GFX9-DPP-NEXT: .LBB9_3: +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s42, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s43, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX1064-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1064-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1064-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX1064-DPP-NEXT: s_cbranch_execz .LBB9_3 +; GFX1064-DPP-NEXT: ; %bb.1: +; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[2:3] +; GFX1064-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1064-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX1064-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1064-DPP-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1064-DPP-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1064-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1064-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1064-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1064-DPP-NEXT: s_clause 0x1 +; GFX1064-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1064-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB9_2 +; GFX1064-DPP-NEXT: .LBB9_3: +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1032-DPP-NEXT: s_mov_b32 s2, exec_lo +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s42, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s43, 0x31c16000 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX1032-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1032-DPP-NEXT: s_mov_b32 s38, 0 +; GFX1032-DPP-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo +; GFX1032-DPP-NEXT: s_cbranch_execz .LBB9_3 +; GFX1032-DPP-NEXT: ; %bb.1: +; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s2 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1032-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1032-DPP-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1032-DPP-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1032-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1032-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1032-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1032-DPP-NEXT: s_clause 0x1 +; GFX1032-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1032-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-DPP-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s38 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB9_2 +; GFX1032-DPP-NEXT: .LBB9_3: +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1164-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-DPP-NEXT: s_cbranch_execz .LBB9_3 +; GFX1164-DPP-NEXT: ; %bb.1: +; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[2:3] +; GFX1164-DPP-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1164-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX1164-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-DPP-NEXT: .p2align 6 +; GFX1164-DPP-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[38:39] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB9_2 +; GFX1164-DPP-NEXT: .LBB9_3: +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_mov_b32 s2, exec_lo +; GFX1132-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s38, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-DPP-NEXT: s_cbranch_execz .LBB9_3 +; GFX1132-DPP-NEXT: ; %bb.1: +; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s2 +; GFX1132-DPP-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1132-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX1132-DPP-NEXT: s_mov_b32 s33, s15 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0 +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-DPP-NEXT: .p2align 6 +; GFX1132-DPP-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s36 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, s37 :: v_dual_mov_b32 v4, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-DPP-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB9_2 +; GFX1132-DPP-NEXT: .LBB9_3: +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-DPP-NEXT: s_endpgm + %result = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("agent") monotonic, align 4 + ret void +} + +define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_movk_i32 s32, 0x800 +; GFX7LESS-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s50, -1 +; GFX7LESS-NEXT: s_mov_b32 s51, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s48, s48, s9 +; GFX7LESS-NEXT: s_addc_u32 s49, s49, 0 +; GFX7LESS-NEXT: s_mov_b32 s33, s8 +; GFX7LESS-NEXT: s_mov_b32 s40, s7 +; GFX7LESS-NEXT: s_mov_b32 s41, s6 +; GFX7LESS-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX7LESS-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX7LESS-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX7LESS-NEXT: s_load_dwordx2 s[44:45], s[2:3], 0x9 +; GFX7LESS-NEXT: s_mov_b32 s47, 0xf000 +; GFX7LESS-NEXT: s_mov_b32 s46, -1 +; GFX7LESS-NEXT: s_add_u32 s8, s36, 44 +; GFX7LESS-NEXT: s_addc_u32 s9, s37, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v42, v0, v2 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX7LESS-NEXT: s_mov_b32 s12, s41 +; GFX7LESS-NEXT: s_mov_b32 s13, s40 +; GFX7LESS-NEXT: s_mov_b32 s14, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v42 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX7LESS-NEXT: v_mov_b32_e32 v40, v0 +; GFX7LESS-NEXT: v_mov_b32_e32 v41, v1 +; GFX7LESS-NEXT: buffer_load_dwordx2 v[0:1], off, s[44:47], 0 +; GFX7LESS-NEXT: s_mov_b64 s[42:43], 0 +; GFX7LESS-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_add_f64 v[2:3], v[0:1], v[40:41] +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:4 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[48:51], 0 +; GFX7LESS-NEXT: s_add_u32 s8, s36, 44 +; GFX7LESS-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:12 +; GFX7LESS-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:8 +; GFX7LESS-NEXT: s_addc_u32 s9, s37, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: s_waitcnt expcnt(2) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, 0 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX7LESS-NEXT: s_mov_b32 s12, s41 +; GFX7LESS-NEXT: s_mov_b32 s13, s40 +; GFX7LESS-NEXT: s_mov_b32 s14, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v42 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s44 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s45 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX7LESS-NEXT: v_and_b32_e32 v2, 1, v0 +; GFX7LESS-NEXT: buffer_load_dword v0, off, s[48:51], 0 +; GFX7LESS-NEXT: buffer_load_dword v1, off, s[48:51], 0 offset:4 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2 +; GFX7LESS-NEXT: s_or_b64 s[42:43], vcc, s[42:43] +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[42:43] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB10_1 +; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s50, -1 +; GFX9-NEXT: s_mov_b32 s51, 0xe00000 +; GFX9-NEXT: s_add_u32 s48, s48, s9 +; GFX9-NEXT: s_addc_u32 s49, s49, 0 +; GFX9-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX9-NEXT: s_mov_b32 s33, s8 +; GFX9-NEXT: s_add_u32 s8, s36, 44 +; GFX9-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX9-NEXT: s_mov_b32 s40, s7 +; GFX9-NEXT: s_mov_b32 s41, s6 +; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX9-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX9-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-NEXT: s_mov_b32 s12, s41 +; GFX9-NEXT: s_mov_b32 s13, s40 +; GFX9-NEXT: s_mov_b32 s14, s33 +; GFX9-NEXT: v_mov_b32_e32 v31, v42 +; GFX9-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-NEXT: s_movk_i32 s32, 0x800 +; GFX9-NEXT: v_mov_b32_e32 v43, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-NEXT: v_mov_b32_e32 v41, v1 +; GFX9-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX9-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-NEXT: s_mov_b64 s[44:45], 0 +; GFX9-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX9-NEXT: s_add_u32 s8, s36, 44 +; GFX9-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX9-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX9-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX9-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX9-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-NEXT: s_mov_b32 s12, s41 +; GFX9-NEXT: s_mov_b32 s13, s40 +; GFX9-NEXT: s_mov_b32 s14, s33 +; GFX9-NEXT: v_mov_b32_e32 v31, v42 +; GFX9-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: v_mov_b32_e32 v2, s42 +; GFX9-NEXT: v_mov_b32_e32 v3, s43 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX9-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX9-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX9-NEXT: s_cbranch_execnz .LBB10_1 +; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s50, -1 +; GFX1064-NEXT: s_mov_b32 s51, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s48, s48, s9 +; GFX1064-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1064-NEXT: s_addc_u32 s49, s49, 0 +; GFX1064-NEXT: s_mov_b32 s33, s8 +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1064-NEXT: s_mov_b32 s40, s7 +; GFX1064-NEXT: s_mov_b32 s41, s6 +; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1064-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1064-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX1064-NEXT: s_mov_b32 s12, s41 +; GFX1064-NEXT: s_mov_b32 s13, s40 +; GFX1064-NEXT: s_mov_b32 s14, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-NEXT: v_mov_b32_e32 v31, v42 +; GFX1064-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-NEXT: v_mov_b32_e32 v43, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-NEXT: v_mov_b32_e32 v41, v1 +; GFX1064-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX1064-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-NEXT: s_mov_b64 s[44:45], 0 +; GFX1064-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX1064-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-NEXT: v_mov_b32_e32 v31, v42 +; GFX1064-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-NEXT: v_mov_b32_e32 v2, s42 +; GFX1064-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-NEXT: s_mov_b32 s12, s41 +; GFX1064-NEXT: s_mov_b32 s13, s40 +; GFX1064-NEXT: s_mov_b32 s14, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX1064-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX1064-NEXT: v_mov_b32_e32 v3, s43 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-NEXT: s_clause 0x1 +; GFX1064-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX1064-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX1064-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX1064-NEXT: s_cbranch_execnz .LBB10_1 +; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s50, -1 +; GFX1032-NEXT: s_mov_b32 s51, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s48, s48, s9 +; GFX1032-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1032-NEXT: s_addc_u32 s49, s49, 0 +; GFX1032-NEXT: s_mov_b32 s33, s8 +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1032-NEXT: s_mov_b32 s40, s7 +; GFX1032-NEXT: s_mov_b32 s41, s6 +; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1032-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1032-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX1032-NEXT: s_mov_b32 s12, s41 +; GFX1032-NEXT: s_mov_b32 s13, s40 +; GFX1032-NEXT: s_mov_b32 s14, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-NEXT: v_mov_b32_e32 v31, v42 +; GFX1032-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-NEXT: v_mov_b32_e32 v43, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-NEXT: v_mov_b32_e32 v41, v1 +; GFX1032-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX1032-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-NEXT: s_mov_b32 s44, 0 +; GFX1032-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX1032-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-NEXT: v_mov_b32_e32 v31, v42 +; GFX1032-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-NEXT: v_mov_b32_e32 v2, s42 +; GFX1032-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-NEXT: s_mov_b32 s12, s41 +; GFX1032-NEXT: s_mov_b32 s13, s40 +; GFX1032-NEXT: s_mov_b32 s14, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX1032-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX1032-NEXT: v_mov_b32_e32 v3, s43 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-NEXT: s_clause 0x1 +; GFX1032-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX1032-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX1032-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s44 +; GFX1032-NEXT: s_cbranch_execnz .LBB10_1 +; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1164-NEXT: s_mov_b32 s33, s8 +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1164-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-NEXT: s_mov_b32 s12, s6 +; GFX1164-NEXT: s_mov_b32 s13, s7 +; GFX1164-NEXT: s_mov_b32 s14, s33 +; GFX1164-NEXT: s_mov_b32 s32, 32 +; GFX1164-NEXT: v_mov_b32_e32 v42, v0 +; GFX1164-NEXT: s_mov_b32 s40, s7 +; GFX1164-NEXT: s_mov_b32 s41, s6 +; GFX1164-NEXT: v_mov_b32_e32 v43, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: v_mov_b32_e32 v41, v1 +; GFX1164-NEXT: global_load_b64 v[1:2], v43, s[42:43] +; GFX1164-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-NEXT: s_mov_b64 s[44:45], 0 +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-NEXT: .p2align 6 +; GFX1164-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-NEXT: v_mov_b32_e32 v31, v42 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-NEXT: s_mov_b32 s12, s41 +; GFX1164-NEXT: s_mov_b32 s13, s40 +; GFX1164-NEXT: s_mov_b32 s14, s33 +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-NEXT: v_mov_b32_e32 v2, s42 +; GFX1164-NEXT: v_mov_b32_e32 v3, s43 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[44:45] +; GFX1164-NEXT: s_cbranch_execnz .LBB10_1 +; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1132-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1132-NEXT: v_mov_b32_e32 v31, v0 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1132-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1132-NEXT: s_mov_b32 s40, s14 +; GFX1132-NEXT: s_mov_b32 s41, s13 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-NEXT: s_mov_b32 s12, s13 +; GFX1132-NEXT: s_mov_b32 s13, s14 +; GFX1132-NEXT: s_mov_b32 s14, s15 +; GFX1132-NEXT: s_mov_b32 s32, 32 +; GFX1132-NEXT: s_mov_b32 s33, s15 +; GFX1132-NEXT: v_dual_mov_b32 v42, v0 :: v_dual_mov_b32 v43, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v41, v1 +; GFX1132-NEXT: global_load_b64 v[1:2], v43, s[42:43] +; GFX1132-NEXT: s_mov_b32 s44, 0 +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-NEXT: .p2align 6 +; GFX1132-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-NEXT: v_dual_mov_b32 v31, v42 :: v_dual_mov_b32 v0, 8 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-NEXT: s_mov_b32 s12, s41 +; GFX1132-NEXT: s_mov_b32 s13, s40 +; GFX1132-NEXT: s_mov_b32 s14, s33 +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s42 +; GFX1132-NEXT: v_dual_mov_b32 v3, s43 :: v_dual_mov_b32 v4, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s44 +; GFX1132-NEXT: s_cbranch_execnz .LBB10_1 +; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s50, -1 +; GFX9-DPP-NEXT: s_mov_b32 s51, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX9-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX9-DPP-NEXT: s_mov_b32 s33, s8 +; GFX9-DPP-NEXT: s_add_u32 s8, s36, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_mov_b32 s40, s7 +; GFX9-DPP-NEXT: s_mov_b32 s41, s6 +; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-DPP-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX9-DPP-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-DPP-NEXT: s_mov_b32 s12, s41 +; GFX9-DPP-NEXT: s_mov_b32 s13, s40 +; GFX9-DPP-NEXT: s_mov_b32 s14, s33 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX9-DPP-NEXT: v_mov_b32_e32 v43, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-DPP-NEXT: v_mov_b32_e32 v41, v1 +; GFX9-DPP-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX9-DPP-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX9-DPP-NEXT: s_add_u32 s8, s36, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX9-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-DPP-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX9-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-DPP-NEXT: s_mov_b32 s12, s41 +; GFX9-DPP-NEXT: s_mov_b32 s13, s40 +; GFX9-DPP-NEXT: s_mov_b32 s14, s33 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-DPP-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX9-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX9-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB10_1 +; GFX9-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s50, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s51, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX1064-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1064-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1064-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-DPP-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX1064-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1064-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v43, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v41, v1 +; GFX1064-DPP-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX1064-DPP-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX1064-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-DPP-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX1064-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-DPP-NEXT: s_clause 0x1 +; GFX1064-DPP-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX1064-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX1064-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB10_1 +; GFX1064-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s50, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s51, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX1032-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1032-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1032-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-DPP-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX1032-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1032-DPP-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v43, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v41, v1 +; GFX1032-DPP-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-DPP-NEXT: s_mov_b32 s44, 0 +; GFX1032-DPP-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX1032-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-DPP-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX1032-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-DPP-NEXT: s_clause 0x1 +; GFX1032-DPP-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX1032-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX1032-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-DPP-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s44 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB10_1 +; GFX1032-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1164-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v42, v0 +; GFX1164-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v43, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v41, v1 +; GFX1164-DPP-NEXT: global_load_b64 v[1:2], v43, s[42:43] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-DPP-NEXT: .p2align 6 +; GFX1164-DPP-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1164-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[44:45] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB10_1 +; GFX1164-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1132-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1132-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1132-DPP-NEXT: s_mov_b32 s40, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s41, s13 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-DPP-NEXT: s_mov_b32 s12, s13 +; GFX1132-DPP-NEXT: s_mov_b32 s13, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s15 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1132-DPP-NEXT: s_mov_b32 s33, s15 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v42, v0 :: v_dual_mov_b32 v43, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v41, v1 +; GFX1132-DPP-NEXT: global_load_b64 v[1:2], v43, s[42:43] +; GFX1132-DPP-NEXT: s_mov_b32 s44, 0 +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-DPP-NEXT: .p2align 6 +; GFX1132-DPP-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v31, v42 :: v_dual_mov_b32 v0, 8 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1132-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s42 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, s43 :: v_dual_mov_b32 v4, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-DPP-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s44 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB10_1 +; GFX1132-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-DPP-NEXT: s_endpgm + %divValue = call double @div.float.value() + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %divValue syncscope("agent") monotonic, align 4 + ret void +} + +define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp(ptr addrspace(1) %ptr) #1 { +; GFX7LESS-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s14, -1 +; GFX7LESS-NEXT: s_mov_b32 s15, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s12, s12, s3 +; GFX7LESS-NEXT: s_addc_u32 s13, s13, 0 +; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec +; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0 +; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX7LESS-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX7LESS-NEXT: s_cbranch_execz .LBB11_3 +; GFX7LESS-NEXT: ; %bb.1: +; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[2:3] +; GFX7LESS-NEXT: s_mov_b32 s7, 0x43300000 +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0 +; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000 +; GFX7LESS-NEXT: v_add_f64 v[0:1], s[6:7], v[0:1] +; GFX7LESS-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s8 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s9 +; GFX7LESS-NEXT: s_mov_b32 s2, -1 +; GFX7LESS-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3 +; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0 +; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3] +; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5] +; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7 +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB11_2 +; GFX7LESS-NEXT: .LBB11_3: +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s10, -1 +; GFX9-NEXT: s_mov_b32 s11, 0xe00000 +; GFX9-NEXT: s_add_u32 s8, s8, s3 +; GFX9-NEXT: s_mov_b64 s[2:3], exec +; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX9-NEXT: s_addc_u32 s9, s9, 0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX9-NEXT: s_cbranch_execz .LBB11_3 +; GFX9-NEXT: ; %bb.1: +; GFX9-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX9-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX9-NEXT: s_mov_b32 s3, 0x43300000 +; GFX9-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1] +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-NEXT: s_mov_b64 s[2:3], 0 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v2, s4 +; GFX9-NEXT: v_mov_b32_e32 v3, s5 +; GFX9-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX9-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX9-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX9-NEXT: s_cbranch_execnz .LBB11_2 +; GFX9-NEXT: .LBB11_3: +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s10, -1 +; GFX1064-NEXT: s_mov_b32 s11, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s8, s8, s3 +; GFX1064-NEXT: s_mov_b64 s[2:3], exec +; GFX1064-NEXT: s_addc_u32 s9, s9, 0 +; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX1064-NEXT: s_cbranch_execz .LBB11_3 +; GFX1064-NEXT: ; %bb.1: +; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX1064-NEXT: s_mov_b32 s3, 0x43300000 +; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1064-NEXT: v_add_f64 v[0:1], 0xc3300000, s[2:3] +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 +; GFX1064-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: v_mov_b32_e32 v2, s2 +; GFX1064-NEXT: v_mov_b32_e32 v3, s3 +; GFX1064-NEXT: s_mov_b64 s[2:3], 0 +; GFX1064-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1064-NEXT: v_mov_b32_e32 v3, v1 +; GFX1064-NEXT: v_mov_b32_e32 v2, v0 +; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX1064-NEXT: s_cbranch_execnz .LBB11_2 +; GFX1064-NEXT: .LBB11_3: +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s10, -1 +; GFX1032-NEXT: s_mov_b32 s11, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s8, s8, s3 +; GFX1032-NEXT: s_mov_b32 s3, exec_lo +; GFX1032-NEXT: s_addc_u32 s9, s9, 0 +; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0 +; GFX1032-NEXT: s_mov_b32 s2, 0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo +; GFX1032-NEXT: s_cbranch_execz .LBB11_3 +; GFX1032-NEXT: ; %bb.1: +; GFX1032-NEXT: s_bcnt1_i32_b32 s4, s3 +; GFX1032-NEXT: s_mov_b32 s5, 0x43300000 +; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[4:5] +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: v_mov_b32_e32 v2, s4 +; GFX1032-NEXT: v_mov_b32_e32 v3, s5 +; GFX1032-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1032-NEXT: v_mov_b32_e32 v3, v1 +; GFX1032-NEXT: v_mov_b32_e32 v2, v0 +; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 +; GFX1032-NEXT: s_cbranch_execnz .LBB11_2 +; GFX1032-NEXT: .LBB11_3: +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_bcnt1_i32_b64 s2, exec +; GFX1164-NEXT: v_mov_b32_e32 v0, 0x43300000 +; GFX1164-NEXT: v_mov_b32_e32 v1, s2 +; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1164-NEXT: s_mov_b64 s[2:3], exec +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b32 off, v0, off offset:4 +; GFX1164-NEXT: scratch_store_b32 off, v1, off +; GFX1164-NEXT: scratch_load_b64 v[0:1], off, off +; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1164-NEXT: s_cbranch_execz .LBB11_3 +; GFX1164-NEXT: ; %bb.1: +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_load_b64 s[2:3], s[0:1], 0x0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1164-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: v_mov_b32_e32 v2, s2 +; GFX1164-NEXT: v_mov_b32_e32 v3, s3 +; GFX1164-NEXT: s_mov_b64 s[2:3], 0 +; GFX1164-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1164-NEXT: v_mov_b32_e32 v3, v1 +; GFX1164-NEXT: v_mov_b32_e32 v2, v0 +; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3] +; GFX1164-NEXT: s_cbranch_execnz .LBB11_2 +; GFX1164-NEXT: .LBB11_3: +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_bcnt1_i32_b32 s2, exec_lo +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s2 +; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1132-NEXT: s_mov_b32 s2, 0 +; GFX1132-NEXT: s_mov_b32 s3, exec_lo +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4 +; GFX1132-NEXT: scratch_store_b32 off, v1, off +; GFX1132-NEXT: scratch_load_b64 v[0:1], off, off +; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1132-NEXT: s_cbranch_execz .LBB11_3 +; GFX1132-NEXT: ; %bb.1: +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1132-NEXT: v_mov_b32_e32 v6, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_load_b64 s[4:5], s[0:1], 0x0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1132-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5 +; GFX1132-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2 +; GFX1132-NEXT: s_cbranch_execnz .LBB11_2 +; GFX1132-NEXT: .LBB11_3: +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s10, -1 +; GFX9-DPP-NEXT: s_mov_b32 s11, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s8, s8, s3 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX9-DPP-NEXT: s_addc_u32 s9, s9, 0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX9-DPP-NEXT: s_cbranch_execz .LBB11_3 +; GFX9-DPP-NEXT: ; %bb.1: +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX9-DPP-NEXT: s_mov_b32 s3, 0x43300000 +; GFX9-DPP-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1] +; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s4 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s5 +; GFX9-DPP-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB11_2 +; GFX9-DPP-NEXT: .LBB11_3: +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s10, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s11, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s8, s8, s3 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX1064-DPP-NEXT: s_addc_u32 s9, s9, 0 +; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX1064-DPP-NEXT: s_cbranch_execz .LBB11_3 +; GFX1064-DPP-NEXT: ; %bb.1: +; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX1064-DPP-NEXT: s_mov_b32 s3, 0x43300000 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1064-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[2:3] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s2 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s3 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX1064-DPP-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB11_2 +; GFX1064-DPP-NEXT: .LBB11_3: +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s10, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s11, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s8, s8, s3 +; GFX1032-DPP-NEXT: s_mov_b32 s3, exec_lo +; GFX1032-DPP-NEXT: s_addc_u32 s9, s9, 0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s2, 0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-DPP-NEXT: s_and_saveexec_b32 s4, vcc_lo +; GFX1032-DPP-NEXT: s_cbranch_execz .LBB11_3 +; GFX1032-DPP-NEXT: ; %bb.1: +; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s4, s3 +; GFX1032-DPP-NEXT: s_mov_b32 s5, 0x43300000 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[4:5] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s4 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s5 +; GFX1032-DPP-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB11_2 +; GFX1032-DPP-NEXT: .LBB11_3: +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s2, exec +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s2 +; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b32 off, v0, off offset:4 +; GFX1164-DPP-NEXT: scratch_store_b32 off, v1, off +; GFX1164-DPP-NEXT: scratch_load_b64 v[0:1], off, off +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1164-DPP-NEXT: s_cbranch_execz .LBB11_3 +; GFX1164-DPP-NEXT: ; %bb.1: +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1164-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s2 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s3 +; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX1164-DPP-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB11_2 +; GFX1164-DPP-NEXT: .LBB11_3: +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s2, exec_lo +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s2 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s2, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s3, exec_lo +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4 +; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off +; GFX1132-DPP-NEXT: scratch_load_b64 v[0:1], off, off +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1132-DPP-NEXT: s_cbranch_execz .LBB11_3 +; GFX1132-DPP-NEXT: ; %bb.1: +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5 +; GFX1132-DPP-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB11_2 +; GFX1132-DPP-NEXT: .LBB11_3: +; GFX1132-DPP-NEXT: s_endpgm + %result = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("one-as") monotonic + ret void +} + +define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_structfp(ptr addrspace(1) %ptr) #1 { +; GFX7LESS-LABEL: global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_mov_b32 s32, 0 +; GFX7LESS-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s42, -1 +; GFX7LESS-NEXT: s_mov_b32 s43, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s40, s40, s9 +; GFX7LESS-NEXT: s_addc_u32 s41, s41, 0 +; GFX7LESS-NEXT: s_mov_b32 s14, s8 +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX7LESS-NEXT: s_load_dwordx2 s[36:37], s[2:3], 0x9 +; GFX7LESS-NEXT: s_mov_b32 s39, 0xf000 +; GFX7LESS-NEXT: s_mov_b32 s38, -1 +; GFX7LESS-NEXT: s_add_u32 s8, s2, 44 +; GFX7LESS-NEXT: s_addc_u32 s9, s3, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[2:3] +; GFX7LESS-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v31, v0, v2 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX7LESS-NEXT: s_mov_b32 s12, s6 +; GFX7LESS-NEXT: s_mov_b32 s13, s7 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX7LESS-NEXT: buffer_load_dwordx2 v[4:5], off, s[36:39], 0 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], 0 +; GFX7LESS-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v9, v5 +; GFX7LESS-NEXT: v_mov_b32_e32 v8, v4 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2 +; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[36:39], 0 glc +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[4:5] +; GFX7LESS-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX7LESS-NEXT: v_mov_b32_e32 v4, v6 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, v7 +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB12_1 +; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s38, -1 +; GFX9-NEXT: s_mov_b32 s39, 0xe00000 +; GFX9-NEXT: s_add_u32 s36, s36, s9 +; GFX9-NEXT: s_addc_u32 s37, s37, 0 +; GFX9-NEXT: s_mov_b32 s14, s8 +; GFX9-NEXT: s_add_u32 s8, s2, 44 +; GFX9-NEXT: s_addc_u32 s9, s3, 0 +; GFX9-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX9-NEXT: s_getpc_b64 s[2:3] +; GFX9-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX9-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX9-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX9-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX9-NEXT: s_mov_b32 s12, s6 +; GFX9-NEXT: s_mov_b32 s13, s7 +; GFX9-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX9-NEXT: s_mov_b32 s32, 0 +; GFX9-NEXT: v_mov_b32_e32 v40, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX9-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX9-NEXT: s_mov_b64 s[0:1], 0 +; GFX9-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX9-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX9-NEXT: v_mov_b32_e32 v5, v3 +; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX9-NEXT: v_mov_b32_e32 v4, v2 +; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX9-NEXT: s_cbranch_execnz .LBB12_1 +; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s38, -1 +; GFX1064-NEXT: s_mov_b32 s39, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s36, s36, s9 +; GFX1064-NEXT: s_addc_u32 s37, s37, 0 +; GFX1064-NEXT: s_mov_b32 s14, s8 +; GFX1064-NEXT: s_add_u32 s8, s2, 44 +; GFX1064-NEXT: s_addc_u32 s9, s3, 0 +; GFX1064-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1064-NEXT: s_getpc_b64 s[4:5] +; GFX1064-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1064-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1064-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1064-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1064-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1064-NEXT: s_mov_b32 s12, s6 +; GFX1064-NEXT: s_mov_b32 s13, s7 +; GFX1064-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1064-NEXT: s_mov_b32 s32, 0 +; GFX1064-NEXT: v_mov_b32_e32 v40, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1064-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1064-NEXT: s_mov_b64 s[0:1], 0 +; GFX1064-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1064-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1064-NEXT: v_mov_b32_e32 v5, v3 +; GFX1064-NEXT: v_mov_b32_e32 v4, v2 +; GFX1064-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX1064-NEXT: s_cbranch_execnz .LBB12_1 +; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s38, -1 +; GFX1032-NEXT: s_mov_b32 s39, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s36, s36, s9 +; GFX1032-NEXT: s_addc_u32 s37, s37, 0 +; GFX1032-NEXT: s_mov_b32 s14, s8 +; GFX1032-NEXT: s_add_u32 s8, s2, 44 +; GFX1032-NEXT: s_addc_u32 s9, s3, 0 +; GFX1032-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1032-NEXT: s_getpc_b64 s[4:5] +; GFX1032-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1032-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1032-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1032-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1032-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1032-NEXT: s_mov_b32 s12, s6 +; GFX1032-NEXT: s_mov_b32 s13, s7 +; GFX1032-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1032-NEXT: s_mov_b32 s32, 0 +; GFX1032-NEXT: v_mov_b32_e32 v40, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1032-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1032-NEXT: s_mov_b32 s0, 0 +; GFX1032-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1032-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1032-NEXT: v_mov_b32_e32 v5, v3 +; GFX1032-NEXT: v_mov_b32_e32 v4, v2 +; GFX1032-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 +; GFX1032-NEXT: s_cbranch_execnz .LBB12_1 +; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b32 s14, s8 +; GFX1164-NEXT: s_add_u32 s8, s2, 44 +; GFX1164-NEXT: s_addc_u32 s9, s3, 0 +; GFX1164-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1164-NEXT: s_getpc_b64 s[4:5] +; GFX1164-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1164-NEXT: s_load_b64 s[16:17], s[4:5], 0x0 +; GFX1164-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1164-NEXT: s_mov_b32 s12, s6 +; GFX1164-NEXT: s_mov_b32 s13, s7 +; GFX1164-NEXT: s_mov_b32 s32, 0 +; GFX1164-NEXT: v_mov_b32_e32 v40, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1164-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1164-NEXT: s_mov_b64 s[0:1], 0 +; GFX1164-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1164-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1164-NEXT: v_mov_b32_e32 v5, v3 +; GFX1164-NEXT: v_mov_b32_e32 v4, v2 +; GFX1164-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[0:1] +; GFX1164-NEXT: s_cbranch_execnz .LBB12_1 +; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_add_u32 s8, s2, 44 +; GFX1132-NEXT: s_addc_u32 s9, s3, 0 +; GFX1132-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1132-NEXT: s_getpc_b64 s[4:5] +; GFX1132-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1132-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 +; GFX1132-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1132-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v31, v0 +; GFX1132-NEXT: s_mov_b32 s12, s13 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1132-NEXT: s_mov_b32 s13, s14 +; GFX1132-NEXT: s_mov_b32 s14, s15 +; GFX1132-NEXT: s_mov_b32 s32, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1132-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1132-NEXT: s_mov_b32 s0, 0 +; GFX1132-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1132-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1132-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2 +; GFX1132-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX1132-NEXT: s_cbranch_execnz .LBB12_1 +; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s38, -1 +; GFX9-DPP-NEXT: s_mov_b32 s39, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX9-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX9-DPP-NEXT: s_mov_b32 s14, s8 +; GFX9-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX9-DPP-NEXT: s_getpc_b64 s[2:3] +; GFX9-DPP-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX9-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX9-DPP-NEXT: s_mov_b32 s12, s6 +; GFX9-DPP-NEXT: s_mov_b32 s13, s7 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX9-DPP-NEXT: s_mov_b32 s32, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX9-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX9-DPP-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX9-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB12_1 +; GFX9-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s38, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s39, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX1064-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1064-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1064-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1064-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1064-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1064-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX1064-DPP-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1064-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB12_1 +; GFX1064-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s38, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s39, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX1032-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1032-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1032-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1032-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1032-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1032-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1032-DPP-NEXT: s_mov_b32 s0, 0 +; GFX1032-DPP-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1032-DPP-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB12_1 +; GFX1032-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1164-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1164-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1164-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: s_load_b64 s[16:17], s[4:5], 0x0 +; GFX1164-DPP-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1164-DPP-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX1164-DPP-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1164-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[0:1] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB12_1 +; GFX1164-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1132-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1132-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 +; GFX1132-DPP-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v31, v0 +; GFX1132-DPP-NEXT: s_mov_b32 s12, s13 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s13, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s15 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1132-DPP-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1132-DPP-NEXT: s_mov_b32 s0, 0 +; GFX1132-DPP-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2 +; GFX1132-DPP-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB12_1 +; GFX1132-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-DPP-NEXT: s_endpgm + %divValue = call double @div.double.value() strictfp + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %divValue syncscope("one-as") monotonic + ret void +} + +define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp(ptr addrspace(1) %ptr) #2{ +; GFX7LESS-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s14, -1 +; GFX7LESS-NEXT: s_mov_b32 s15, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s12, s12, s3 +; GFX7LESS-NEXT: s_addc_u32 s13, s13, 0 +; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec +; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0 +; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX7LESS-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX7LESS-NEXT: s_cbranch_execz .LBB13_3 +; GFX7LESS-NEXT: ; %bb.1: +; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[2:3] +; GFX7LESS-NEXT: s_mov_b32 s7, 0x43300000 +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0 +; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000 +; GFX7LESS-NEXT: v_add_f64 v[0:1], s[6:7], v[0:1] +; GFX7LESS-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s8 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s9 +; GFX7LESS-NEXT: s_mov_b32 s2, -1 +; GFX7LESS-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3 +; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0 +; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3] +; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5] +; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7 +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB13_2 +; GFX7LESS-NEXT: .LBB13_3: +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s10, -1 +; GFX9-NEXT: s_mov_b32 s11, 0xe00000 +; GFX9-NEXT: s_add_u32 s8, s8, s3 +; GFX9-NEXT: s_mov_b64 s[2:3], exec +; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX9-NEXT: s_addc_u32 s9, s9, 0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX9-NEXT: s_cbranch_execz .LBB13_3 +; GFX9-NEXT: ; %bb.1: +; GFX9-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX9-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX9-NEXT: s_mov_b32 s3, 0x43300000 +; GFX9-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1] +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-NEXT: s_mov_b64 s[2:3], 0 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v2, s4 +; GFX9-NEXT: v_mov_b32_e32 v3, s5 +; GFX9-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX9-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX9-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX9-NEXT: s_cbranch_execnz .LBB13_2 +; GFX9-NEXT: .LBB13_3: +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s10, -1 +; GFX1064-NEXT: s_mov_b32 s11, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s8, s8, s3 +; GFX1064-NEXT: s_mov_b64 s[2:3], exec +; GFX1064-NEXT: s_addc_u32 s9, s9, 0 +; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX1064-NEXT: s_cbranch_execz .LBB13_3 +; GFX1064-NEXT: ; %bb.1: +; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX1064-NEXT: s_mov_b32 s3, 0x43300000 +; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1064-NEXT: v_add_f64 v[0:1], 0xc3300000, s[2:3] +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 +; GFX1064-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: v_mov_b32_e32 v2, s2 +; GFX1064-NEXT: v_mov_b32_e32 v3, s3 +; GFX1064-NEXT: s_mov_b64 s[2:3], 0 +; GFX1064-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1064-NEXT: v_mov_b32_e32 v3, v1 +; GFX1064-NEXT: v_mov_b32_e32 v2, v0 +; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX1064-NEXT: s_cbranch_execnz .LBB13_2 +; GFX1064-NEXT: .LBB13_3: +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s10, -1 +; GFX1032-NEXT: s_mov_b32 s11, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s8, s8, s3 +; GFX1032-NEXT: s_mov_b32 s3, exec_lo +; GFX1032-NEXT: s_addc_u32 s9, s9, 0 +; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0 +; GFX1032-NEXT: s_mov_b32 s2, 0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo +; GFX1032-NEXT: s_cbranch_execz .LBB13_3 +; GFX1032-NEXT: ; %bb.1: +; GFX1032-NEXT: s_bcnt1_i32_b32 s4, s3 +; GFX1032-NEXT: s_mov_b32 s5, 0x43300000 +; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[4:5] +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: v_mov_b32_e32 v2, s4 +; GFX1032-NEXT: v_mov_b32_e32 v3, s5 +; GFX1032-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1032-NEXT: v_mov_b32_e32 v3, v1 +; GFX1032-NEXT: v_mov_b32_e32 v2, v0 +; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 +; GFX1032-NEXT: s_cbranch_execnz .LBB13_2 +; GFX1032-NEXT: .LBB13_3: +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_bcnt1_i32_b64 s2, exec +; GFX1164-NEXT: v_mov_b32_e32 v0, 0x43300000 +; GFX1164-NEXT: v_mov_b32_e32 v1, s2 +; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1164-NEXT: s_mov_b64 s[2:3], exec +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b32 off, v0, off offset:4 +; GFX1164-NEXT: scratch_store_b32 off, v1, off +; GFX1164-NEXT: scratch_load_b64 v[0:1], off, off +; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1164-NEXT: s_cbranch_execz .LBB13_3 +; GFX1164-NEXT: ; %bb.1: +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_load_b64 s[2:3], s[0:1], 0x0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1164-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: v_mov_b32_e32 v2, s2 +; GFX1164-NEXT: v_mov_b32_e32 v3, s3 +; GFX1164-NEXT: s_mov_b64 s[2:3], 0 +; GFX1164-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1164-NEXT: v_mov_b32_e32 v3, v1 +; GFX1164-NEXT: v_mov_b32_e32 v2, v0 +; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3] +; GFX1164-NEXT: s_cbranch_execnz .LBB13_2 +; GFX1164-NEXT: .LBB13_3: +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_bcnt1_i32_b32 s2, exec_lo +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s2 +; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1132-NEXT: s_mov_b32 s2, 0 +; GFX1132-NEXT: s_mov_b32 s3, exec_lo +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4 +; GFX1132-NEXT: scratch_store_b32 off, v1, off +; GFX1132-NEXT: scratch_load_b64 v[0:1], off, off +; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1132-NEXT: s_cbranch_execz .LBB13_3 +; GFX1132-NEXT: ; %bb.1: +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1132-NEXT: v_mov_b32_e32 v6, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_load_b64 s[4:5], s[0:1], 0x0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1132-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5 +; GFX1132-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2 +; GFX1132-NEXT: s_cbranch_execnz .LBB13_2 +; GFX1132-NEXT: .LBB13_3: +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s10, -1 +; GFX9-DPP-NEXT: s_mov_b32 s11, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s8, s8, s3 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX9-DPP-NEXT: s_addc_u32 s9, s9, 0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX9-DPP-NEXT: s_cbranch_execz .LBB13_3 +; GFX9-DPP-NEXT: ; %bb.1: +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX9-DPP-NEXT: s_mov_b32 s3, 0x43300000 +; GFX9-DPP-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1] +; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s4 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s5 +; GFX9-DPP-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB13_2 +; GFX9-DPP-NEXT: .LBB13_3: +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s10, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s11, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s8, s8, s3 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX1064-DPP-NEXT: s_addc_u32 s9, s9, 0 +; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX1064-DPP-NEXT: s_cbranch_execz .LBB13_3 +; GFX1064-DPP-NEXT: ; %bb.1: +; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX1064-DPP-NEXT: s_mov_b32 s3, 0x43300000 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1064-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[2:3] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s2 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s3 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX1064-DPP-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB13_2 +; GFX1064-DPP-NEXT: .LBB13_3: +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s10, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s11, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s8, s8, s3 +; GFX1032-DPP-NEXT: s_mov_b32 s3, exec_lo +; GFX1032-DPP-NEXT: s_addc_u32 s9, s9, 0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s2, 0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-DPP-NEXT: s_and_saveexec_b32 s4, vcc_lo +; GFX1032-DPP-NEXT: s_cbranch_execz .LBB13_3 +; GFX1032-DPP-NEXT: ; %bb.1: +; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s4, s3 +; GFX1032-DPP-NEXT: s_mov_b32 s5, 0x43300000 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[4:5] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s4 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s5 +; GFX1032-DPP-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB13_2 +; GFX1032-DPP-NEXT: .LBB13_3: +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s2, exec +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s2 +; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b32 off, v0, off offset:4 +; GFX1164-DPP-NEXT: scratch_store_b32 off, v1, off +; GFX1164-DPP-NEXT: scratch_load_b64 v[0:1], off, off +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1164-DPP-NEXT: s_cbranch_execz .LBB13_3 +; GFX1164-DPP-NEXT: ; %bb.1: +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1164-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s2 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s3 +; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX1164-DPP-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB13_2 +; GFX1164-DPP-NEXT: .LBB13_3: +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s2, exec_lo +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s2 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s2, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s3, exec_lo +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4 +; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off +; GFX1132-DPP-NEXT: scratch_load_b64 v[0:1], off, off +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1132-DPP-NEXT: s_cbranch_execz .LBB13_3 +; GFX1132-DPP-NEXT: ; %bb.1: +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5 +; GFX1132-DPP-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB13_2 +; GFX1132-DPP-NEXT: .LBB13_3: +; GFX1132-DPP-NEXT: s_endpgm + %result = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("agent") monotonic + ret void +} + +define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_mov_b32 s32, 0 +; GFX7LESS-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s42, -1 +; GFX7LESS-NEXT: s_mov_b32 s43, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s40, s40, s9 +; GFX7LESS-NEXT: s_addc_u32 s41, s41, 0 +; GFX7LESS-NEXT: s_mov_b32 s14, s8 +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX7LESS-NEXT: s_load_dwordx2 s[36:37], s[2:3], 0x9 +; GFX7LESS-NEXT: s_mov_b32 s39, 0xf000 +; GFX7LESS-NEXT: s_mov_b32 s38, -1 +; GFX7LESS-NEXT: s_add_u32 s8, s2, 44 +; GFX7LESS-NEXT: s_addc_u32 s9, s3, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[2:3] +; GFX7LESS-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v31, v0, v2 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX7LESS-NEXT: s_mov_b32 s12, s6 +; GFX7LESS-NEXT: s_mov_b32 s13, s7 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX7LESS-NEXT: buffer_load_dwordx2 v[4:5], off, s[36:39], 0 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], 0 +; GFX7LESS-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v9, v5 +; GFX7LESS-NEXT: v_mov_b32_e32 v8, v4 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2 +; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[36:39], 0 glc +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[4:5] +; GFX7LESS-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX7LESS-NEXT: v_mov_b32_e32 v4, v6 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, v7 +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB14_1 +; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s38, -1 +; GFX9-NEXT: s_mov_b32 s39, 0xe00000 +; GFX9-NEXT: s_add_u32 s36, s36, s9 +; GFX9-NEXT: s_addc_u32 s37, s37, 0 +; GFX9-NEXT: s_mov_b32 s14, s8 +; GFX9-NEXT: s_add_u32 s8, s2, 44 +; GFX9-NEXT: s_addc_u32 s9, s3, 0 +; GFX9-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX9-NEXT: s_getpc_b64 s[2:3] +; GFX9-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX9-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX9-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX9-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX9-NEXT: s_mov_b32 s12, s6 +; GFX9-NEXT: s_mov_b32 s13, s7 +; GFX9-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX9-NEXT: s_mov_b32 s32, 0 +; GFX9-NEXT: v_mov_b32_e32 v40, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX9-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX9-NEXT: s_mov_b64 s[0:1], 0 +; GFX9-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX9-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX9-NEXT: v_mov_b32_e32 v5, v3 +; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX9-NEXT: v_mov_b32_e32 v4, v2 +; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX9-NEXT: s_cbranch_execnz .LBB14_1 +; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s38, -1 +; GFX1064-NEXT: s_mov_b32 s39, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s36, s36, s9 +; GFX1064-NEXT: s_addc_u32 s37, s37, 0 +; GFX1064-NEXT: s_mov_b32 s14, s8 +; GFX1064-NEXT: s_add_u32 s8, s2, 44 +; GFX1064-NEXT: s_addc_u32 s9, s3, 0 +; GFX1064-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1064-NEXT: s_getpc_b64 s[4:5] +; GFX1064-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1064-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1064-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1064-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1064-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1064-NEXT: s_mov_b32 s12, s6 +; GFX1064-NEXT: s_mov_b32 s13, s7 +; GFX1064-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1064-NEXT: s_mov_b32 s32, 0 +; GFX1064-NEXT: v_mov_b32_e32 v40, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1064-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1064-NEXT: s_mov_b64 s[0:1], 0 +; GFX1064-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1064-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1064-NEXT: v_mov_b32_e32 v5, v3 +; GFX1064-NEXT: v_mov_b32_e32 v4, v2 +; GFX1064-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX1064-NEXT: s_cbranch_execnz .LBB14_1 +; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s38, -1 +; GFX1032-NEXT: s_mov_b32 s39, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s36, s36, s9 +; GFX1032-NEXT: s_addc_u32 s37, s37, 0 +; GFX1032-NEXT: s_mov_b32 s14, s8 +; GFX1032-NEXT: s_add_u32 s8, s2, 44 +; GFX1032-NEXT: s_addc_u32 s9, s3, 0 +; GFX1032-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1032-NEXT: s_getpc_b64 s[4:5] +; GFX1032-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1032-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1032-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1032-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1032-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1032-NEXT: s_mov_b32 s12, s6 +; GFX1032-NEXT: s_mov_b32 s13, s7 +; GFX1032-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1032-NEXT: s_mov_b32 s32, 0 +; GFX1032-NEXT: v_mov_b32_e32 v40, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1032-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1032-NEXT: s_mov_b32 s0, 0 +; GFX1032-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1032-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1032-NEXT: v_mov_b32_e32 v5, v3 +; GFX1032-NEXT: v_mov_b32_e32 v4, v2 +; GFX1032-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 +; GFX1032-NEXT: s_cbranch_execnz .LBB14_1 +; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b32 s14, s8 +; GFX1164-NEXT: s_add_u32 s8, s2, 44 +; GFX1164-NEXT: s_addc_u32 s9, s3, 0 +; GFX1164-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1164-NEXT: s_getpc_b64 s[4:5] +; GFX1164-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1164-NEXT: s_load_b64 s[16:17], s[4:5], 0x0 +; GFX1164-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1164-NEXT: s_mov_b32 s12, s6 +; GFX1164-NEXT: s_mov_b32 s13, s7 +; GFX1164-NEXT: s_mov_b32 s32, 0 +; GFX1164-NEXT: v_mov_b32_e32 v40, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1164-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1164-NEXT: s_mov_b64 s[0:1], 0 +; GFX1164-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1164-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1164-NEXT: v_mov_b32_e32 v5, v3 +; GFX1164-NEXT: v_mov_b32_e32 v4, v2 +; GFX1164-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[0:1] +; GFX1164-NEXT: s_cbranch_execnz .LBB14_1 +; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_add_u32 s8, s2, 44 +; GFX1132-NEXT: s_addc_u32 s9, s3, 0 +; GFX1132-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1132-NEXT: s_getpc_b64 s[4:5] +; GFX1132-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1132-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 +; GFX1132-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1132-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v31, v0 +; GFX1132-NEXT: s_mov_b32 s12, s13 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1132-NEXT: s_mov_b32 s13, s14 +; GFX1132-NEXT: s_mov_b32 s14, s15 +; GFX1132-NEXT: s_mov_b32 s32, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1132-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1132-NEXT: s_mov_b32 s0, 0 +; GFX1132-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1132-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1132-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2 +; GFX1132-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX1132-NEXT: s_cbranch_execnz .LBB14_1 +; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s38, -1 +; GFX9-DPP-NEXT: s_mov_b32 s39, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX9-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX9-DPP-NEXT: s_mov_b32 s14, s8 +; GFX9-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX9-DPP-NEXT: s_getpc_b64 s[2:3] +; GFX9-DPP-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX9-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX9-DPP-NEXT: s_mov_b32 s12, s6 +; GFX9-DPP-NEXT: s_mov_b32 s13, s7 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX9-DPP-NEXT: s_mov_b32 s32, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX9-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX9-DPP-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX9-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB14_1 +; GFX9-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s38, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s39, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX1064-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1064-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1064-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1064-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1064-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1064-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX1064-DPP-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1064-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB14_1 +; GFX1064-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s38, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s39, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX1032-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1032-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1032-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1032-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1032-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1032-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1032-DPP-NEXT: s_mov_b32 s0, 0 +; GFX1032-DPP-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1032-DPP-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB14_1 +; GFX1032-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1164-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1164-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1164-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: s_load_b64 s[16:17], s[4:5], 0x0 +; GFX1164-DPP-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1164-DPP-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX1164-DPP-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1164-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[0:1] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB14_1 +; GFX1164-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1132-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1132-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 +; GFX1132-DPP-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v31, v0 +; GFX1132-DPP-NEXT: s_mov_b32 s12, s13 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s13, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s15 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1132-DPP-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1132-DPP-NEXT: s_mov_b32 s0, 0 +; GFX1132-DPP-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2 +; GFX1132-DPP-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB14_1 +; GFX1132-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-DPP-NEXT: s_endpgm + %divValue = call double @div.double.value() + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %divValue syncscope("agent") monotonic + ret void +} + +define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe_structfp(ptr addrspace(1) %ptr) #1 { +; GFX7LESS-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_mov_b32 s32, 0 +; GFX7LESS-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s42, -1 +; GFX7LESS-NEXT: s_mov_b32 s43, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s40, s40, s9 +; GFX7LESS-NEXT: s_addc_u32 s41, s41, 0 +; GFX7LESS-NEXT: s_mov_b32 s14, s8 +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX7LESS-NEXT: s_load_dwordx2 s[36:37], s[2:3], 0x9 +; GFX7LESS-NEXT: s_mov_b32 s39, 0xf000 +; GFX7LESS-NEXT: s_mov_b32 s38, -1 +; GFX7LESS-NEXT: s_add_u32 s8, s2, 44 +; GFX7LESS-NEXT: s_addc_u32 s9, s3, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[2:3] +; GFX7LESS-NEXT: s_add_u32 s2, s2, div.float.value@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s3, s3, div.float.value@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v31, v0, v2 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX7LESS-NEXT: s_mov_b32 s12, s6 +; GFX7LESS-NEXT: s_mov_b32 s13, s7 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX7LESS-NEXT: buffer_load_dwordx2 v[4:5], off, s[36:39], 0 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], 0 +; GFX7LESS-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v9, v5 +; GFX7LESS-NEXT: v_mov_b32_e32 v8, v4 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2 +; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[36:39], 0 glc +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[4:5] +; GFX7LESS-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX7LESS-NEXT: v_mov_b32_e32 v4, v6 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, v7 +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB15_1 +; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s38, -1 +; GFX9-NEXT: s_mov_b32 s39, 0xe00000 +; GFX9-NEXT: s_add_u32 s36, s36, s9 +; GFX9-NEXT: s_addc_u32 s37, s37, 0 +; GFX9-NEXT: s_mov_b32 s14, s8 +; GFX9-NEXT: s_add_u32 s8, s2, 44 +; GFX9-NEXT: s_addc_u32 s9, s3, 0 +; GFX9-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX9-NEXT: s_getpc_b64 s[2:3] +; GFX9-NEXT: s_add_u32 s2, s2, div.float.value@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s3, s3, div.float.value@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX9-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX9-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX9-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX9-NEXT: s_mov_b32 s12, s6 +; GFX9-NEXT: s_mov_b32 s13, s7 +; GFX9-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX9-NEXT: s_mov_b32 s32, 0 +; GFX9-NEXT: v_mov_b32_e32 v40, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX9-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX9-NEXT: s_mov_b64 s[0:1], 0 +; GFX9-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX9-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX9-NEXT: v_mov_b32_e32 v5, v3 +; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX9-NEXT: v_mov_b32_e32 v4, v2 +; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX9-NEXT: s_cbranch_execnz .LBB15_1 +; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s38, -1 +; GFX1064-NEXT: s_mov_b32 s39, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s36, s36, s9 +; GFX1064-NEXT: s_addc_u32 s37, s37, 0 +; GFX1064-NEXT: s_mov_b32 s14, s8 +; GFX1064-NEXT: s_add_u32 s8, s2, 44 +; GFX1064-NEXT: s_addc_u32 s9, s3, 0 +; GFX1064-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1064-NEXT: s_getpc_b64 s[4:5] +; GFX1064-NEXT: s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12 +; GFX1064-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1064-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1064-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1064-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1064-NEXT: s_mov_b32 s12, s6 +; GFX1064-NEXT: s_mov_b32 s13, s7 +; GFX1064-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1064-NEXT: s_mov_b32 s32, 0 +; GFX1064-NEXT: v_mov_b32_e32 v40, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1064-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1064-NEXT: s_mov_b64 s[0:1], 0 +; GFX1064-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1064-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1064-NEXT: v_mov_b32_e32 v5, v3 +; GFX1064-NEXT: v_mov_b32_e32 v4, v2 +; GFX1064-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX1064-NEXT: s_cbranch_execnz .LBB15_1 +; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s38, -1 +; GFX1032-NEXT: s_mov_b32 s39, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s36, s36, s9 +; GFX1032-NEXT: s_addc_u32 s37, s37, 0 +; GFX1032-NEXT: s_mov_b32 s14, s8 +; GFX1032-NEXT: s_add_u32 s8, s2, 44 +; GFX1032-NEXT: s_addc_u32 s9, s3, 0 +; GFX1032-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1032-NEXT: s_getpc_b64 s[4:5] +; GFX1032-NEXT: s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12 +; GFX1032-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1032-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1032-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1032-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1032-NEXT: s_mov_b32 s12, s6 +; GFX1032-NEXT: s_mov_b32 s13, s7 +; GFX1032-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1032-NEXT: s_mov_b32 s32, 0 +; GFX1032-NEXT: v_mov_b32_e32 v40, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1032-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1032-NEXT: s_mov_b32 s0, 0 +; GFX1032-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1032-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1032-NEXT: v_mov_b32_e32 v5, v3 +; GFX1032-NEXT: v_mov_b32_e32 v4, v2 +; GFX1032-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 +; GFX1032-NEXT: s_cbranch_execnz .LBB15_1 +; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b32 s14, s8 +; GFX1164-NEXT: s_add_u32 s8, s2, 44 +; GFX1164-NEXT: s_addc_u32 s9, s3, 0 +; GFX1164-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1164-NEXT: s_getpc_b64 s[4:5] +; GFX1164-NEXT: s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12 +; GFX1164-NEXT: s_load_b64 s[16:17], s[4:5], 0x0 +; GFX1164-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1164-NEXT: s_mov_b32 s12, s6 +; GFX1164-NEXT: s_mov_b32 s13, s7 +; GFX1164-NEXT: s_mov_b32 s32, 0 +; GFX1164-NEXT: v_mov_b32_e32 v40, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1164-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1164-NEXT: s_mov_b64 s[0:1], 0 +; GFX1164-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1164-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1164-NEXT: v_mov_b32_e32 v5, v3 +; GFX1164-NEXT: v_mov_b32_e32 v4, v2 +; GFX1164-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[0:1] +; GFX1164-NEXT: s_cbranch_execnz .LBB15_1 +; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_add_u32 s8, s2, 44 +; GFX1132-NEXT: s_addc_u32 s9, s3, 0 +; GFX1132-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1132-NEXT: s_getpc_b64 s[4:5] +; GFX1132-NEXT: s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12 +; GFX1132-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 +; GFX1132-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1132-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v31, v0 +; GFX1132-NEXT: s_mov_b32 s12, s13 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1132-NEXT: s_mov_b32 s13, s14 +; GFX1132-NEXT: s_mov_b32 s14, s15 +; GFX1132-NEXT: s_mov_b32 s32, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1132-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1132-NEXT: s_mov_b32 s0, 0 +; GFX1132-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1132-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1132-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2 +; GFX1132-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX1132-NEXT: s_cbranch_execnz .LBB15_1 +; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s38, -1 +; GFX9-DPP-NEXT: s_mov_b32 s39, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX9-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX9-DPP-NEXT: s_mov_b32 s14, s8 +; GFX9-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX9-DPP-NEXT: s_getpc_b64 s[2:3] +; GFX9-DPP-NEXT: s_add_u32 s2, s2, div.float.value@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s3, s3, div.float.value@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX9-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX9-DPP-NEXT: s_mov_b32 s12, s6 +; GFX9-DPP-NEXT: s_mov_b32 s13, s7 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX9-DPP-NEXT: s_mov_b32 s32, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX9-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX9-DPP-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX9-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB15_1 +; GFX9-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s38, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s39, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX1064-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1064-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1064-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1064-DPP-NEXT: s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1064-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1064-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX1064-DPP-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1064-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB15_1 +; GFX1064-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s38, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s39, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX1032-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1032-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1032-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1032-DPP-NEXT: s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1032-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1032-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1032-DPP-NEXT: s_mov_b32 s0, 0 +; GFX1032-DPP-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1032-DPP-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB15_1 +; GFX1032-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1164-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1164-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1164-DPP-NEXT: s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: s_load_b64 s[16:17], s[4:5], 0x0 +; GFX1164-DPP-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1164-DPP-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX1164-DPP-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1164-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[0:1] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB15_1 +; GFX1164-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1132-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1132-DPP-NEXT: s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 +; GFX1132-DPP-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v31, v0 +; GFX1132-DPP-NEXT: s_mov_b32 s12, s13 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s13, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s15 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1132-DPP-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1132-DPP-NEXT: s_mov_b32 s0, 0 +; GFX1132-DPP-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2 +; GFX1132-DPP-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB15_1 +; GFX1132-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-DPP-NEXT: s_endpgm + %divValue = call double @div.float.value() strictfp + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %divValue syncscope("agent") monotonic + ret void +} + +define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defalut_scope_strictfp(ptr addrspace(1) %ptr) #2 { +; GFX7LESS-LABEL: global_atomic_fadd_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_movk_i32 s32, 0x800 +; GFX7LESS-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s42, -1 +; GFX7LESS-NEXT: s_mov_b32 s43, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s40, s40, s3 +; GFX7LESS-NEXT: s_addc_u32 s41, s41, 0 +; GFX7LESS-NEXT: s_mov_b32 s33, s2 +; GFX7LESS-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX7LESS-NEXT: v_mov_b32_e32 v40, v0 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec +; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s0, 0 +; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s1, v0 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX7LESS-NEXT: s_cbranch_execz .LBB16_3 +; GFX7LESS-NEXT: ; %bb.1: +; GFX7LESS-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x9 +; GFX7LESS-NEXT: s_bcnt1_i32_b64 s0, s[0:1] +; GFX7LESS-NEXT: s_mov_b32 s1, 0x43300000 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_load_dwordx2 s[2:3], s[36:37], 0x0 +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX7LESS-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1] +; GFX7LESS-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX7LESS-NEXT: s_mov_b64 s[38:39], 0 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, s2 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, s3 +; GFX7LESS-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_add_f64 v[2:3], v[0:1], v[41:42] +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[40:43], 0 offset:4 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[40:43], 0 +; GFX7LESS-NEXT: s_add_u32 s8, s34, 44 +; GFX7LESS-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:12 +; GFX7LESS-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:8 +; GFX7LESS-NEXT: s_addc_u32 s9, s35, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX7LESS-NEXT: s_waitcnt expcnt(2) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, 0 +; GFX7LESS-NEXT: s_mov_b32 s12, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v40 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s36 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s37 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX7LESS-NEXT: v_and_b32_e32 v2, 1, v0 +; GFX7LESS-NEXT: buffer_load_dword v0, off, s[40:43], 0 +; GFX7LESS-NEXT: buffer_load_dword v1, off, s[40:43], 0 offset:4 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2 +; GFX7LESS-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB16_2 +; GFX7LESS-NEXT: .LBB16_3: +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fadd_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s42, -1 +; GFX9-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX9-NEXT: s_mov_b64 s[0:1], exec +; GFX9-NEXT: s_mov_b32 s43, 0xe00000 +; GFX9-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0 +; GFX9-NEXT: s_add_u32 s40, s40, s3 +; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0 +; GFX9-NEXT: s_addc_u32 s41, s41, 0 +; GFX9-NEXT: s_mov_b32 s33, s2 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-NEXT: s_movk_i32 s32, 0x800 +; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX9-NEXT: s_cbranch_execz .LBB16_3 +; GFX9-NEXT: ; %bb.1: +; GFX9-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1] +; GFX9-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX9-NEXT: s_mov_b32 s1, 0x43300000 +; GFX9-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1] +; GFX9-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX9-NEXT: s_mov_b64 s[38:39], 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX9-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX9-NEXT: s_add_u32 s8, s34, 44 +; GFX9-NEXT: s_addc_u32 s9, s35, 0 +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX9-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX9-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX9-NEXT: s_mov_b32 s12, s33 +; GFX9-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX9-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX9-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX9-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: v_mov_b32_e32 v2, s36 +; GFX9-NEXT: v_mov_b32_e32 v3, s37 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX9-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX9-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX9-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX9-NEXT: s_cbranch_execnz .LBB16_2 +; GFX9-NEXT: .LBB16_3: +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fadd_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s42, -1 +; GFX1064-NEXT: s_mov_b32 s43, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s40, s40, s3 +; GFX1064-NEXT: s_mov_b32 s33, s2 +; GFX1064-NEXT: s_mov_b64 s[2:3], exec +; GFX1064-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1064-NEXT: s_addc_u32 s41, s41, 0 +; GFX1064-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1064-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX1064-NEXT: s_cbranch_execz .LBB16_3 +; GFX1064-NEXT: ; %bb.1: +; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[2:3] +; GFX1064-NEXT: s_mov_b32 s1, 0x43300000 +; GFX1064-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1064-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1] +; GFX1064-NEXT: s_mov_b64 s[38:39], 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1064-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: v_mov_b32_e32 v2, s1 +; GFX1064-NEXT: v_mov_b32_e32 v1, s0 +; GFX1064-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1064-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1064-NEXT: s_mov_b32 s12, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1064-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1064-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1064-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1064-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-NEXT: v_mov_b32_e32 v2, s36 +; GFX1064-NEXT: v_mov_b32_e32 v3, s37 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1064-NEXT: s_clause 0x1 +; GFX1064-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1064-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX1064-NEXT: s_cbranch_execnz .LBB16_2 +; GFX1064-NEXT: .LBB16_3: +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fadd_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s33, s2 +; GFX1032-NEXT: s_mov_b32 s2, exec_lo +; GFX1032-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1032-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s42, -1 +; GFX1032-NEXT: s_mov_b32 s43, 0x31c16000 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-NEXT: s_add_u32 s40, s40, s3 +; GFX1032-NEXT: s_addc_u32 s41, s41, 0 +; GFX1032-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1032-NEXT: s_mov_b32 s38, 0 +; GFX1032-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo +; GFX1032-NEXT: s_cbranch_execz .LBB16_3 +; GFX1032-NEXT: ; %bb.1: +; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s2 +; GFX1032-NEXT: s_mov_b32 s1, 0x43300000 +; GFX1032-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1] +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1032-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: v_mov_b32_e32 v2, s1 +; GFX1032-NEXT: v_mov_b32_e32 v1, s0 +; GFX1032-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1032-NEXT: s_mov_b32 s12, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1032-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1032-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1032-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1032-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-NEXT: v_mov_b32_e32 v2, s36 +; GFX1032-NEXT: v_mov_b32_e32 v3, s37 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1032-NEXT: s_clause 0x1 +; GFX1032-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1032-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s38 +; GFX1032-NEXT: s_cbranch_execnz .LBB16_2 +; GFX1032-NEXT: .LBB16_3: +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fadd_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1164-NEXT: s_bcnt1_i32_b64 s0, exec +; GFX1164-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-NEXT: v_mov_b32_e32 v0, 0x43300000 +; GFX1164-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b32 off, v0, off offset:20 +; GFX1164-NEXT: scratch_store_b32 off, v1, off offset:16 +; GFX1164-NEXT: scratch_load_b64 v[0:1], off, off offset:16 +; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 +; GFX1164-NEXT: s_mov_b32 s32, 32 +; GFX1164-NEXT: s_mov_b64 s[0:1], exec +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1164-NEXT: s_cbranch_execz .LBB16_3 +; GFX1164-NEXT: ; %bb.1: +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1164-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1164-NEXT: s_mov_b32 s33, s2 +; GFX1164-NEXT: s_mov_b64 s[38:39], 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: v_mov_b32_e32 v2, s1 +; GFX1164-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-NEXT: .p2align 6 +; GFX1164-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-NEXT: s_mov_b32 s12, s33 +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-NEXT: v_mov_b32_e32 v2, s36 +; GFX1164-NEXT: v_mov_b32_e32 v3, s37 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[38:39] +; GFX1164-NEXT: s_cbranch_execnz .LBB16_2 +; GFX1164-NEXT: .LBB16_3: +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fadd_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v1, s0 +; GFX1132-NEXT: v_mov_b32_e32 v0, 0x43300000 +; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:20 +; GFX1132-NEXT: scratch_store_b32 off, v1, off offset:16 +; GFX1132-NEXT: scratch_load_b64 v[0:1], off, off offset:16 +; GFX1132-NEXT: s_mov_b32 s38, 0 +; GFX1132-NEXT: s_mov_b32 s32, 32 +; GFX1132-NEXT: s_mov_b32 s0, exec_lo +; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1132-NEXT: s_cbranch_execz .LBB16_3 +; GFX1132-NEXT: ; %bb.1: +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1132-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1132-NEXT: s_mov_b32 s33, s15 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0 +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-NEXT: .p2align 6 +; GFX1132-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-NEXT: v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-NEXT: s_mov_b32 s12, s33 +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s36 +; GFX1132-NEXT: v_dual_mov_b32 v3, s37 :: v_dual_mov_b32 v4, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38 +; GFX1132-NEXT: s_cbranch_execnz .LBB16_2 +; GFX1132-NEXT: .LBB16_3: +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s42, -1 +; GFX9-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], exec +; GFX9-DPP-NEXT: s_mov_b32 s43, 0xe00000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0 +; GFX9-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0 +; GFX9-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX9-DPP-NEXT: s_mov_b32 s33, s2 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX9-DPP-NEXT: s_cbranch_execz .LBB16_3 +; GFX9-DPP-NEXT: ; %bb.1: +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1] +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX9-DPP-NEXT: s_mov_b32 s1, 0x43300000 +; GFX9-DPP-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1] +; GFX9-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX9-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX9-DPP-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-DPP-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX9-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX9-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX9-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX9-DPP-NEXT: s_mov_b32 s12, s33 +; GFX9-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX9-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX9-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX9-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX9-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB16_2 +; GFX9-DPP-NEXT: .LBB16_3: +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s42, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s43, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX1064-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1064-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1064-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX1064-DPP-NEXT: s_cbranch_execz .LBB16_3 +; GFX1064-DPP-NEXT: ; %bb.1: +; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[2:3] +; GFX1064-DPP-NEXT: s_mov_b32 s1, 0x43300000 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1064-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1] +; GFX1064-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1064-DPP-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1064-DPP-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1064-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1064-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1064-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1064-DPP-NEXT: s_clause 0x1 +; GFX1064-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1064-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB16_2 +; GFX1064-DPP-NEXT: .LBB16_3: +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1032-DPP-NEXT: s_mov_b32 s2, exec_lo +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s42, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s43, 0x31c16000 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX1032-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1032-DPP-NEXT: s_mov_b32 s38, 0 +; GFX1032-DPP-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo +; GFX1032-DPP-NEXT: s_cbranch_execz .LBB16_3 +; GFX1032-DPP-NEXT: ; %bb.1: +; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s2 +; GFX1032-DPP-NEXT: s_mov_b32 s1, 0x43300000 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1] +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1032-DPP-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1032-DPP-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1032-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1032-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1032-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1032-DPP-NEXT: s_clause 0x1 +; GFX1032-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1032-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-DPP-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s38 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB16_2 +; GFX1032-DPP-NEXT: .LBB16_3: +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, exec +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b32 off, v0, off offset:20 +; GFX1164-DPP-NEXT: scratch_store_b32 off, v1, off offset:16 +; GFX1164-DPP-NEXT: scratch_load_b64 v[0:1], off, off offset:16 +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1164-DPP-NEXT: s_cbranch_execz .LBB16_3 +; GFX1164-DPP-NEXT: ; %bb.1: +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1164-DPP-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1164-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1164-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-DPP-NEXT: .p2align 6 +; GFX1164-DPP-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[38:39] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB16_2 +; GFX1164-DPP-NEXT: .LBB16_3: +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v1, s0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:20 +; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off offset:16 +; GFX1132-DPP-NEXT: scratch_load_b64 v[0:1], off, off offset:16 +; GFX1132-DPP-NEXT: s_mov_b32 s38, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1132-DPP-NEXT: s_cbranch_execz .LBB16_3 +; GFX1132-DPP-NEXT: ; %bb.1: +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1132-DPP-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1132-DPP-NEXT: s_mov_b32 s33, s15 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0 +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-DPP-NEXT: .p2align 6 +; GFX1132-DPP-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s36 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, s37 :: v_dual_mov_b32 v4, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-DPP-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB16_2 +; GFX1132-DPP-NEXT: .LBB16_3: +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-DPP-NEXT: s_endpgm + %result = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 monotonic, align 4 + ret void +} + +define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defalut_scope_strictfp(ptr addrspace(1) %ptr) #2 { +; GFX7LESS-LABEL: global_atomic_fadd_double_uni_address_div_value_defalut_scope_strictfp: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_movk_i32 s32, 0x800 +; GFX7LESS-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s50, -1 +; GFX7LESS-NEXT: s_mov_b32 s51, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s48, s48, s9 +; GFX7LESS-NEXT: s_addc_u32 s49, s49, 0 +; GFX7LESS-NEXT: s_mov_b32 s33, s8 +; GFX7LESS-NEXT: s_mov_b32 s40, s7 +; GFX7LESS-NEXT: s_mov_b32 s41, s6 +; GFX7LESS-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX7LESS-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX7LESS-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX7LESS-NEXT: s_load_dwordx2 s[44:45], s[2:3], 0x9 +; GFX7LESS-NEXT: s_mov_b32 s47, 0xf000 +; GFX7LESS-NEXT: s_mov_b32 s46, -1 +; GFX7LESS-NEXT: s_add_u32 s8, s36, 44 +; GFX7LESS-NEXT: s_addc_u32 s9, s37, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v42, v0, v2 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX7LESS-NEXT: s_mov_b32 s12, s41 +; GFX7LESS-NEXT: s_mov_b32 s13, s40 +; GFX7LESS-NEXT: s_mov_b32 s14, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v42 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX7LESS-NEXT: v_mov_b32_e32 v40, v0 +; GFX7LESS-NEXT: v_mov_b32_e32 v41, v1 +; GFX7LESS-NEXT: buffer_load_dwordx2 v[0:1], off, s[44:47], 0 +; GFX7LESS-NEXT: s_mov_b64 s[42:43], 0 +; GFX7LESS-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_add_f64 v[2:3], v[0:1], v[40:41] +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:4 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[48:51], 0 +; GFX7LESS-NEXT: s_add_u32 s8, s36, 44 +; GFX7LESS-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:12 +; GFX7LESS-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:8 +; GFX7LESS-NEXT: s_addc_u32 s9, s37, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: s_waitcnt expcnt(2) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, 0 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX7LESS-NEXT: s_mov_b32 s12, s41 +; GFX7LESS-NEXT: s_mov_b32 s13, s40 +; GFX7LESS-NEXT: s_mov_b32 s14, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v42 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s44 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s45 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX7LESS-NEXT: v_and_b32_e32 v2, 1, v0 +; GFX7LESS-NEXT: buffer_load_dword v0, off, s[48:51], 0 +; GFX7LESS-NEXT: buffer_load_dword v1, off, s[48:51], 0 offset:4 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2 +; GFX7LESS-NEXT: s_or_b64 s[42:43], vcc, s[42:43] +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[42:43] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB17_1 +; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fadd_double_uni_address_div_value_defalut_scope_strictfp: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s50, -1 +; GFX9-NEXT: s_mov_b32 s51, 0xe00000 +; GFX9-NEXT: s_add_u32 s48, s48, s9 +; GFX9-NEXT: s_addc_u32 s49, s49, 0 +; GFX9-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX9-NEXT: s_mov_b32 s33, s8 +; GFX9-NEXT: s_add_u32 s8, s36, 44 +; GFX9-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX9-NEXT: s_mov_b32 s40, s7 +; GFX9-NEXT: s_mov_b32 s41, s6 +; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX9-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX9-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-NEXT: s_mov_b32 s12, s41 +; GFX9-NEXT: s_mov_b32 s13, s40 +; GFX9-NEXT: s_mov_b32 s14, s33 +; GFX9-NEXT: v_mov_b32_e32 v31, v42 +; GFX9-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-NEXT: s_movk_i32 s32, 0x800 +; GFX9-NEXT: v_mov_b32_e32 v43, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-NEXT: v_mov_b32_e32 v41, v1 +; GFX9-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX9-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-NEXT: s_mov_b64 s[44:45], 0 +; GFX9-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX9-NEXT: s_add_u32 s8, s36, 44 +; GFX9-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX9-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX9-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX9-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX9-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-NEXT: s_mov_b32 s12, s41 +; GFX9-NEXT: s_mov_b32 s13, s40 +; GFX9-NEXT: s_mov_b32 s14, s33 +; GFX9-NEXT: v_mov_b32_e32 v31, v42 +; GFX9-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: v_mov_b32_e32 v2, s42 +; GFX9-NEXT: v_mov_b32_e32 v3, s43 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX9-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX9-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX9-NEXT: s_cbranch_execnz .LBB17_1 +; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fadd_double_uni_address_div_value_defalut_scope_strictfp: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s50, -1 +; GFX1064-NEXT: s_mov_b32 s51, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s48, s48, s9 +; GFX1064-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1064-NEXT: s_addc_u32 s49, s49, 0 +; GFX1064-NEXT: s_mov_b32 s33, s8 +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1064-NEXT: s_mov_b32 s40, s7 +; GFX1064-NEXT: s_mov_b32 s41, s6 +; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1064-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1064-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX1064-NEXT: s_mov_b32 s12, s41 +; GFX1064-NEXT: s_mov_b32 s13, s40 +; GFX1064-NEXT: s_mov_b32 s14, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-NEXT: v_mov_b32_e32 v31, v42 +; GFX1064-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-NEXT: v_mov_b32_e32 v43, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-NEXT: v_mov_b32_e32 v41, v1 +; GFX1064-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX1064-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-NEXT: s_mov_b64 s[44:45], 0 +; GFX1064-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX1064-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-NEXT: v_mov_b32_e32 v31, v42 +; GFX1064-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-NEXT: v_mov_b32_e32 v2, s42 +; GFX1064-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-NEXT: s_mov_b32 s12, s41 +; GFX1064-NEXT: s_mov_b32 s13, s40 +; GFX1064-NEXT: s_mov_b32 s14, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX1064-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX1064-NEXT: v_mov_b32_e32 v3, s43 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-NEXT: s_clause 0x1 +; GFX1064-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX1064-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX1064-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX1064-NEXT: s_cbranch_execnz .LBB17_1 +; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fadd_double_uni_address_div_value_defalut_scope_strictfp: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s50, -1 +; GFX1032-NEXT: s_mov_b32 s51, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s48, s48, s9 +; GFX1032-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1032-NEXT: s_addc_u32 s49, s49, 0 +; GFX1032-NEXT: s_mov_b32 s33, s8 +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1032-NEXT: s_mov_b32 s40, s7 +; GFX1032-NEXT: s_mov_b32 s41, s6 +; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1032-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1032-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX1032-NEXT: s_mov_b32 s12, s41 +; GFX1032-NEXT: s_mov_b32 s13, s40 +; GFX1032-NEXT: s_mov_b32 s14, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-NEXT: v_mov_b32_e32 v31, v42 +; GFX1032-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-NEXT: v_mov_b32_e32 v43, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-NEXT: v_mov_b32_e32 v41, v1 +; GFX1032-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX1032-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-NEXT: s_mov_b32 s44, 0 +; GFX1032-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX1032-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-NEXT: v_mov_b32_e32 v31, v42 +; GFX1032-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-NEXT: v_mov_b32_e32 v2, s42 +; GFX1032-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-NEXT: s_mov_b32 s12, s41 +; GFX1032-NEXT: s_mov_b32 s13, s40 +; GFX1032-NEXT: s_mov_b32 s14, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX1032-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX1032-NEXT: v_mov_b32_e32 v3, s43 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-NEXT: s_clause 0x1 +; GFX1032-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX1032-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX1032-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s44 +; GFX1032-NEXT: s_cbranch_execnz .LBB17_1 +; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fadd_double_uni_address_div_value_defalut_scope_strictfp: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1164-NEXT: s_mov_b32 s33, s8 +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1164-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-NEXT: s_mov_b32 s12, s6 +; GFX1164-NEXT: s_mov_b32 s13, s7 +; GFX1164-NEXT: s_mov_b32 s14, s33 +; GFX1164-NEXT: s_mov_b32 s32, 32 +; GFX1164-NEXT: v_mov_b32_e32 v42, v0 +; GFX1164-NEXT: s_mov_b32 s40, s7 +; GFX1164-NEXT: s_mov_b32 s41, s6 +; GFX1164-NEXT: v_mov_b32_e32 v43, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: v_mov_b32_e32 v41, v1 +; GFX1164-NEXT: global_load_b64 v[1:2], v43, s[42:43] +; GFX1164-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-NEXT: s_mov_b64 s[44:45], 0 +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-NEXT: .p2align 6 +; GFX1164-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-NEXT: v_mov_b32_e32 v31, v42 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-NEXT: s_mov_b32 s12, s41 +; GFX1164-NEXT: s_mov_b32 s13, s40 +; GFX1164-NEXT: s_mov_b32 s14, s33 +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-NEXT: v_mov_b32_e32 v2, s42 +; GFX1164-NEXT: v_mov_b32_e32 v3, s43 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[44:45] +; GFX1164-NEXT: s_cbranch_execnz .LBB17_1 +; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fadd_double_uni_address_div_value_defalut_scope_strictfp: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1132-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1132-NEXT: v_mov_b32_e32 v31, v0 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1132-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1132-NEXT: s_mov_b32 s40, s14 +; GFX1132-NEXT: s_mov_b32 s41, s13 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-NEXT: s_mov_b32 s12, s13 +; GFX1132-NEXT: s_mov_b32 s13, s14 +; GFX1132-NEXT: s_mov_b32 s14, s15 +; GFX1132-NEXT: s_mov_b32 s32, 32 +; GFX1132-NEXT: s_mov_b32 s33, s15 +; GFX1132-NEXT: v_dual_mov_b32 v42, v0 :: v_dual_mov_b32 v43, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v41, v1 +; GFX1132-NEXT: global_load_b64 v[1:2], v43, s[42:43] +; GFX1132-NEXT: s_mov_b32 s44, 0 +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-NEXT: .p2align 6 +; GFX1132-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-NEXT: v_dual_mov_b32 v31, v42 :: v_dual_mov_b32 v0, 8 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-NEXT: s_mov_b32 s12, s41 +; GFX1132-NEXT: s_mov_b32 s13, s40 +; GFX1132-NEXT: s_mov_b32 s14, s33 +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s42 +; GFX1132-NEXT: v_dual_mov_b32 v3, s43 :: v_dual_mov_b32 v4, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s44 +; GFX1132-NEXT: s_cbranch_execnz .LBB17_1 +; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_defalut_scope_strictfp: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s50, -1 +; GFX9-DPP-NEXT: s_mov_b32 s51, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX9-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX9-DPP-NEXT: s_mov_b32 s33, s8 +; GFX9-DPP-NEXT: s_add_u32 s8, s36, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_mov_b32 s40, s7 +; GFX9-DPP-NEXT: s_mov_b32 s41, s6 +; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-DPP-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX9-DPP-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-DPP-NEXT: s_mov_b32 s12, s41 +; GFX9-DPP-NEXT: s_mov_b32 s13, s40 +; GFX9-DPP-NEXT: s_mov_b32 s14, s33 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX9-DPP-NEXT: v_mov_b32_e32 v43, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-DPP-NEXT: v_mov_b32_e32 v41, v1 +; GFX9-DPP-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX9-DPP-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX9-DPP-NEXT: s_add_u32 s8, s36, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX9-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-DPP-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX9-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-DPP-NEXT: s_mov_b32 s12, s41 +; GFX9-DPP-NEXT: s_mov_b32 s13, s40 +; GFX9-DPP-NEXT: s_mov_b32 s14, s33 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-DPP-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX9-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX9-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB17_1 +; GFX9-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_defalut_scope_strictfp: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s50, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s51, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX1064-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1064-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1064-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-DPP-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX1064-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1064-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v43, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v41, v1 +; GFX1064-DPP-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX1064-DPP-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX1064-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-DPP-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX1064-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-DPP-NEXT: s_clause 0x1 +; GFX1064-DPP-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX1064-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX1064-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB17_1 +; GFX1064-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_defalut_scope_strictfp: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s50, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s51, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX1032-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1032-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1032-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-DPP-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX1032-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1032-DPP-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v43, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v41, v1 +; GFX1032-DPP-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-DPP-NEXT: s_mov_b32 s44, 0 +; GFX1032-DPP-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX1032-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-DPP-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX1032-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-DPP-NEXT: s_clause 0x1 +; GFX1032-DPP-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX1032-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX1032-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-DPP-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s44 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB17_1 +; GFX1032-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_defalut_scope_strictfp: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1164-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v42, v0 +; GFX1164-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v43, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v41, v1 +; GFX1164-DPP-NEXT: global_load_b64 v[1:2], v43, s[42:43] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-DPP-NEXT: .p2align 6 +; GFX1164-DPP-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1164-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[44:45] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB17_1 +; GFX1164-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_defalut_scope_strictfp: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1132-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1132-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1132-DPP-NEXT: s_mov_b32 s40, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s41, s13 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-DPP-NEXT: s_mov_b32 s12, s13 +; GFX1132-DPP-NEXT: s_mov_b32 s13, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s15 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1132-DPP-NEXT: s_mov_b32 s33, s15 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v42, v0 :: v_dual_mov_b32 v43, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v41, v1 +; GFX1132-DPP-NEXT: global_load_b64 v[1:2], v43, s[42:43] +; GFX1132-DPP-NEXT: s_mov_b32 s44, 0 +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-DPP-NEXT: .p2align 6 +; GFX1132-DPP-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v31, v42 :: v_dual_mov_b32 v0, 8 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1132-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s42 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, s43 :: v_dual_mov_b32 v4, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-DPP-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s44 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB17_1 +; GFX1132-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-DPP-NEXT: s_endpgm + %divValue = call double @div.float.value() strictfp + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %divValue monotonic, align 4 + ret void +} + attributes #0 = { "denormal-fp-math-f32"="preserve-sign,preserve-sign" "amdgpu-unsafe-fp-atomics"="true" } attributes #1 = { strictfp "denormal-fp-math-f32"="preserve-sign,preserve-sign" "amdgpu-unsafe-fp-atomics"="true" } attributes #2 = { strictfp} diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll index 3cc5a4cd1d0a..622be43e7442 100644 --- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll +++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll @@ -13,6 +13,7 @@ ; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,-wavefrontsize64 -amdgpu-atomic-optimizer-strategy=DPP -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX1132-DPP %s declare float @div.float.value() +declare float @div.double.value() define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 { ; GFX7LESS-LABEL: global_atomic_fmax_uni_address_uni_value_agent_scope_unsafe: @@ -3550,6 +3551,3965 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_defalut_scop ret void } +define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_movk_i32 s32, 0x800 +; GFX7LESS-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s42, -1 +; GFX7LESS-NEXT: s_mov_b32 s43, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s40, s40, s3 +; GFX7LESS-NEXT: s_addc_u32 s41, s41, 0 +; GFX7LESS-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX7LESS-NEXT: v_mov_b32_e32 v40, v0 +; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0 +; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX7LESS-NEXT: s_cbranch_execz .LBB6_3 +; GFX7LESS-NEXT: ; %bb.1: +; GFX7LESS-NEXT: s_mov_b32 s33, s2 +; GFX7LESS-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x9 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX7LESS-NEXT: s_mov_b64 s[38:39], 0 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, s0 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, s1 +; GFX7LESS-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_max_f64 v[2:3], v[0:1], v[0:1] +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[40:43], 0 offset:4 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[40:43], 0 +; GFX7LESS-NEXT: s_add_u32 s8, s34, 44 +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_max_f64 v[0:1], v[2:3], 4.0 +; GFX7LESS-NEXT: s_addc_u32 s9, s35, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[40:43], 0 offset:12 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[40:43], 0 offset:8 +; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, 0 +; GFX7LESS-NEXT: s_mov_b32 s12, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v40 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s36 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s37 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX7LESS-NEXT: v_and_b32_e32 v2, 1, v0 +; GFX7LESS-NEXT: buffer_load_dword v0, off, s[40:43], 0 +; GFX7LESS-NEXT: buffer_load_dword v1, off, s[40:43], 0 offset:4 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2 +; GFX7LESS-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB6_2 +; GFX7LESS-NEXT: .LBB6_3: +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s42, -1 +; GFX9-NEXT: s_mov_b32 s43, 0xe00000 +; GFX9-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX9-NEXT: s_add_u32 s40, s40, s3 +; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-NEXT: s_addc_u32 s41, s41, 0 +; GFX9-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-NEXT: s_movk_i32 s32, 0x800 +; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX9-NEXT: s_cbranch_execz .LBB6_3 +; GFX9-NEXT: ; %bb.1: +; GFX9-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX9-NEXT: s_mov_b32 s33, s2 +; GFX9-NEXT: s_mov_b64 s[38:39], 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX9-NEXT: s_add_u32 s8, s34, 44 +; GFX9-NEXT: s_addc_u32 s9, s35, 0 +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX9-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX9-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX9-NEXT: s_mov_b32 s12, s33 +; GFX9-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX9-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX9-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: v_mov_b32_e32 v2, s36 +; GFX9-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX9-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX9-NEXT: v_mov_b32_e32 v3, s37 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX9-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX9-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX9-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX9-NEXT: s_cbranch_execnz .LBB6_2 +; GFX9-NEXT: .LBB6_3: +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1064-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s42, -1 +; GFX1064-NEXT: s_mov_b32 s43, 0x31e16000 +; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1064-NEXT: s_add_u32 s40, s40, s3 +; GFX1064-NEXT: s_addc_u32 s41, s41, 0 +; GFX1064-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1064-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX1064-NEXT: s_cbranch_execz .LBB6_3 +; GFX1064-NEXT: ; %bb.1: +; GFX1064-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1064-NEXT: s_mov_b32 s33, s2 +; GFX1064-NEXT: s_mov_b64 s[38:39], 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: v_mov_b32_e32 v2, s1 +; GFX1064-NEXT: v_mov_b32_e32 v1, s0 +; GFX1064-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1064-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1064-NEXT: s_mov_b32 s12, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1064-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX1064-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1064-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1064-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1064-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-NEXT: v_mov_b32_e32 v2, s36 +; GFX1064-NEXT: v_mov_b32_e32 v3, s37 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1064-NEXT: s_clause 0x1 +; GFX1064-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1064-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX1064-NEXT: s_cbranch_execnz .LBB6_2 +; GFX1064-NEXT: .LBB6_3: +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1032-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s42, -1 +; GFX1032-NEXT: s_mov_b32 s43, 0x31c16000 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-NEXT: s_add_u32 s40, s40, s3 +; GFX1032-NEXT: s_addc_u32 s41, s41, 0 +; GFX1032-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1032-NEXT: s_mov_b32 s38, 0 +; GFX1032-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo +; GFX1032-NEXT: s_cbranch_execz .LBB6_3 +; GFX1032-NEXT: ; %bb.1: +; GFX1032-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1032-NEXT: s_mov_b32 s33, s2 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: v_mov_b32_e32 v2, s1 +; GFX1032-NEXT: v_mov_b32_e32 v1, s0 +; GFX1032-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1032-NEXT: s_mov_b32 s12, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1032-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX1032-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1032-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1032-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1032-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-NEXT: v_mov_b32_e32 v2, s36 +; GFX1032-NEXT: v_mov_b32_e32 v3, s37 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1032-NEXT: s_clause 0x1 +; GFX1032-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1032-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s38 +; GFX1032-NEXT: s_cbranch_execnz .LBB6_2 +; GFX1032-NEXT: .LBB6_3: +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1164-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1164-NEXT: s_mov_b32 s32, 32 +; GFX1164-NEXT: s_mov_b64 s[0:1], exec +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-NEXT: s_cbranch_execz .LBB6_3 +; GFX1164-NEXT: ; %bb.1: +; GFX1164-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1164-NEXT: s_mov_b32 s33, s2 +; GFX1164-NEXT: s_mov_b64 s[38:39], 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: v_mov_b32_e32 v2, s1 +; GFX1164-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-NEXT: .p2align 6 +; GFX1164-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-NEXT: s_mov_b32 s12, s33 +; GFX1164-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-NEXT: v_mov_b32_e32 v2, s36 +; GFX1164-NEXT: v_mov_b32_e32 v3, s37 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[38:39] +; GFX1164-NEXT: s_cbranch_execnz .LBB6_2 +; GFX1164-NEXT: .LBB6_3: +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: v_mov_b32_e32 v40, v0 +; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1132-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1132-NEXT: s_mov_b32 s38, 0 +; GFX1132-NEXT: s_mov_b32 s32, 32 +; GFX1132-NEXT: s_mov_b32 s0, exec_lo +; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-NEXT: s_cbranch_execz .LBB6_3 +; GFX1132-NEXT: ; %bb.1: +; GFX1132-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1132-NEXT: s_mov_b32 s33, s15 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0 +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-NEXT: .p2align 6 +; GFX1132-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-NEXT: v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-NEXT: s_mov_b32 s12, s33 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX1132-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s36 +; GFX1132-NEXT: v_dual_mov_b32 v3, s37 :: v_dual_mov_b32 v4, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38 +; GFX1132-NEXT: s_cbranch_execnz .LBB6_2 +; GFX1132-NEXT: .LBB6_3: +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s42, -1 +; GFX9-DPP-NEXT: s_mov_b32 s43, 0xe00000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX9-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX9-DPP-NEXT: s_cbranch_execz .LBB6_3 +; GFX9-DPP-NEXT: ; %bb.1: +; GFX9-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX9-DPP-NEXT: s_mov_b32 s33, s2 +; GFX9-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-DPP-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX9-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX9-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX9-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX9-DPP-NEXT: s_mov_b32 s12, s33 +; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX9-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX9-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX9-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX9-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB6_2 +; GFX9-DPP-NEXT: .LBB6_3: +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s42, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s43, 0x31e16000 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1064-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX1064-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1064-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX1064-DPP-NEXT: s_cbranch_execz .LBB6_3 +; GFX1064-DPP-NEXT: ; %bb.1: +; GFX1064-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1064-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1064-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1064-DPP-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX1064-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1064-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1064-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1064-DPP-NEXT: s_clause 0x1 +; GFX1064-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1064-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB6_2 +; GFX1064-DPP-NEXT: .LBB6_3: +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s42, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s43, 0x31c16000 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX1032-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1032-DPP-NEXT: s_mov_b32 s38, 0 +; GFX1032-DPP-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo +; GFX1032-DPP-NEXT: s_cbranch_execz .LBB6_3 +; GFX1032-DPP-NEXT: ; %bb.1: +; GFX1032-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1032-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1032-DPP-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX1032-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1032-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1032-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1032-DPP-NEXT: s_clause 0x1 +; GFX1032-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1032-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-DPP-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s38 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB6_2 +; GFX1032-DPP-NEXT: .LBB6_3: +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1164-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-DPP-NEXT: s_cbranch_execz .LBB6_3 +; GFX1164-DPP-NEXT: ; %bb.1: +; GFX1164-DPP-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1164-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1164-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-DPP-NEXT: .p2align 6 +; GFX1164-DPP-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1164-DPP-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[38:39] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB6_2 +; GFX1164-DPP-NEXT: .LBB6_3: +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s38, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-DPP-NEXT: s_cbranch_execz .LBB6_3 +; GFX1132-DPP-NEXT: ; %bb.1: +; GFX1132-DPP-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1132-DPP-NEXT: s_mov_b32 s33, s15 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0 +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-DPP-NEXT: .p2align 6 +; GFX1132-DPP-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX1132-DPP-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s36 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, s37 :: v_dual_mov_b32 v4, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-DPP-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB6_2 +; GFX1132-DPP-NEXT: .LBB6_3: +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-DPP-NEXT: s_endpgm + %result = atomicrmw fmax ptr addrspace(1) %ptr, double 4.0 syncscope("agent") monotonic, align 4 + ret void +} + +define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_movk_i32 s32, 0x800 +; GFX7LESS-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s50, -1 +; GFX7LESS-NEXT: s_mov_b32 s51, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s48, s48, s9 +; GFX7LESS-NEXT: s_addc_u32 s49, s49, 0 +; GFX7LESS-NEXT: s_mov_b32 s33, s8 +; GFX7LESS-NEXT: s_mov_b32 s40, s7 +; GFX7LESS-NEXT: s_mov_b32 s41, s6 +; GFX7LESS-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX7LESS-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX7LESS-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX7LESS-NEXT: s_load_dwordx2 s[44:45], s[2:3], 0x9 +; GFX7LESS-NEXT: s_mov_b32 s47, 0xf000 +; GFX7LESS-NEXT: s_mov_b32 s46, -1 +; GFX7LESS-NEXT: s_add_u32 s8, s36, 44 +; GFX7LESS-NEXT: s_addc_u32 s9, s37, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v40, v0, v2 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX7LESS-NEXT: s_mov_b32 s12, s41 +; GFX7LESS-NEXT: s_mov_b32 s13, s40 +; GFX7LESS-NEXT: s_mov_b32 s14, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v40 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX7LESS-NEXT: buffer_load_dwordx2 v[2:3], off, s[44:47], 0 +; GFX7LESS-NEXT: s_mov_b64 s[42:43], 0 +; GFX7LESS-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX7LESS-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX7LESS-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX7LESS-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX7LESS-NEXT: s_add_u32 s8, s36, 44 +; GFX7LESS-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX7LESS-NEXT: s_addc_u32 s9, s37, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, 0 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX7LESS-NEXT: s_mov_b32 s12, s41 +; GFX7LESS-NEXT: s_mov_b32 s13, s40 +; GFX7LESS-NEXT: s_mov_b32 s14, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v40 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s44 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s45 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX7LESS-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX7LESS-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX7LESS-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX7LESS-NEXT: s_or_b64 s[42:43], vcc, s[42:43] +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[42:43] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB7_1 +; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s50, -1 +; GFX9-NEXT: s_mov_b32 s51, 0xe00000 +; GFX9-NEXT: s_add_u32 s48, s48, s9 +; GFX9-NEXT: s_addc_u32 s49, s49, 0 +; GFX9-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX9-NEXT: s_mov_b32 s33, s8 +; GFX9-NEXT: s_add_u32 s8, s36, 44 +; GFX9-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX9-NEXT: s_mov_b32 s40, s7 +; GFX9-NEXT: s_mov_b32 s41, s6 +; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX9-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX9-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-NEXT: s_mov_b32 s12, s41 +; GFX9-NEXT: s_mov_b32 s13, s40 +; GFX9-NEXT: s_mov_b32 s14, s33 +; GFX9-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-NEXT: s_movk_i32 s32, 0x800 +; GFX9-NEXT: v_mov_b32_e32 v41, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX9-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX9-NEXT: s_mov_b64 s[44:45], 0 +; GFX9-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX9-NEXT: s_add_u32 s8, s36, 44 +; GFX9-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX9-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX9-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX9-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-NEXT: s_mov_b32 s12, s41 +; GFX9-NEXT: s_mov_b32 s13, s40 +; GFX9-NEXT: s_mov_b32 s14, s33 +; GFX9-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-NEXT: v_mov_b32_e32 v2, s42 +; GFX9-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX9-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX9-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: v_mov_b32_e32 v3, s43 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX9-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX9-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX9-NEXT: s_cbranch_execnz .LBB7_1 +; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s50, -1 +; GFX1064-NEXT: s_mov_b32 s51, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s48, s48, s9 +; GFX1064-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1064-NEXT: s_addc_u32 s49, s49, 0 +; GFX1064-NEXT: s_mov_b32 s33, s8 +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1064-NEXT: s_mov_b32 s40, s7 +; GFX1064-NEXT: s_mov_b32 s41, s6 +; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1064-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1064-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX1064-NEXT: s_mov_b32 s12, s41 +; GFX1064-NEXT: s_mov_b32 s13, s40 +; GFX1064-NEXT: s_mov_b32 s14, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-NEXT: v_mov_b32_e32 v41, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX1064-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1064-NEXT: s_mov_b64 s[44:45], 0 +; GFX1064-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX1064-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-NEXT: v_mov_b32_e32 v2, s42 +; GFX1064-NEXT: v_mov_b32_e32 v3, s43 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-NEXT: s_mov_b32 s12, s41 +; GFX1064-NEXT: s_mov_b32 s13, s40 +; GFX1064-NEXT: s_mov_b32 s14, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX1064-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX1064-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX1064-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-NEXT: s_clause 0x1 +; GFX1064-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX1064-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX1064-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX1064-NEXT: s_cbranch_execnz .LBB7_1 +; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s50, -1 +; GFX1032-NEXT: s_mov_b32 s51, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s48, s48, s9 +; GFX1032-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1032-NEXT: s_addc_u32 s49, s49, 0 +; GFX1032-NEXT: s_mov_b32 s33, s8 +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1032-NEXT: s_mov_b32 s40, s7 +; GFX1032-NEXT: s_mov_b32 s41, s6 +; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1032-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1032-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX1032-NEXT: s_mov_b32 s12, s41 +; GFX1032-NEXT: s_mov_b32 s13, s40 +; GFX1032-NEXT: s_mov_b32 s14, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-NEXT: v_mov_b32_e32 v41, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX1032-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1032-NEXT: s_mov_b32 s44, 0 +; GFX1032-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX1032-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-NEXT: v_mov_b32_e32 v2, s42 +; GFX1032-NEXT: v_mov_b32_e32 v3, s43 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-NEXT: s_mov_b32 s12, s41 +; GFX1032-NEXT: s_mov_b32 s13, s40 +; GFX1032-NEXT: s_mov_b32 s14, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX1032-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX1032-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX1032-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-NEXT: s_clause 0x1 +; GFX1032-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX1032-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX1032-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s44 +; GFX1032-NEXT: s_cbranch_execnz .LBB7_1 +; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1164-NEXT: s_mov_b32 s33, s8 +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1164-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-NEXT: s_mov_b32 s12, s6 +; GFX1164-NEXT: s_mov_b32 s13, s7 +; GFX1164-NEXT: s_mov_b32 s14, s33 +; GFX1164-NEXT: s_mov_b32 s32, 32 +; GFX1164-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-NEXT: s_mov_b32 s40, s7 +; GFX1164-NEXT: s_mov_b32 s41, s6 +; GFX1164-NEXT: v_mov_b32_e32 v41, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: global_load_b64 v[2:3], v41, s[42:43] +; GFX1164-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1164-NEXT: s_mov_b64 s[44:45], 0 +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-NEXT: .p2align 6 +; GFX1164-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-NEXT: s_mov_b32 s12, s41 +; GFX1164-NEXT: s_mov_b32 s13, s40 +; GFX1164-NEXT: s_mov_b32 s14, s33 +; GFX1164-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b64 off, v[2:3], off +; GFX1164-NEXT: scratch_store_b64 off, v[0:1], off offset:8 +; GFX1164-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-NEXT: v_mov_b32_e32 v2, s42 +; GFX1164-NEXT: v_mov_b32_e32 v3, s43 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: scratch_load_b64 v[2:3], off, off +; GFX1164-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[44:45] +; GFX1164-NEXT: s_cbranch_execnz .LBB7_1 +; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1132-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1132-NEXT: v_mov_b32_e32 v31, v0 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1132-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1132-NEXT: s_mov_b32 s40, s14 +; GFX1132-NEXT: s_mov_b32 s41, s13 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-NEXT: s_mov_b32 s12, s13 +; GFX1132-NEXT: s_mov_b32 s13, s14 +; GFX1132-NEXT: s_mov_b32 s14, s15 +; GFX1132-NEXT: s_mov_b32 s32, 32 +; GFX1132-NEXT: s_mov_b32 s33, s15 +; GFX1132-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v41, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: global_load_b64 v[2:3], v41, s[42:43] +; GFX1132-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1132-NEXT: s_mov_b32 s44, 0 +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-NEXT: .p2align 6 +; GFX1132-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-NEXT: v_mov_b32_e32 v31, v40 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-NEXT: s_mov_b32 s12, s41 +; GFX1132-NEXT: s_mov_b32 s13, s40 +; GFX1132-NEXT: s_mov_b32 s14, s33 +; GFX1132-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b64 off, v[2:3], off +; GFX1132-NEXT: scratch_store_b64 off, v[0:1], off offset:8 +; GFX1132-NEXT: v_dual_mov_b32 v0, 8 :: v_dual_mov_b32 v1, 0 +; GFX1132-NEXT: v_dual_mov_b32 v2, s42 :: v_dual_mov_b32 v3, s43 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: scratch_load_b64 v[2:3], off, off +; GFX1132-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s44 +; GFX1132-NEXT: s_cbranch_execnz .LBB7_1 +; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s50, -1 +; GFX9-DPP-NEXT: s_mov_b32 s51, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX9-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX9-DPP-NEXT: s_mov_b32 s33, s8 +; GFX9-DPP-NEXT: s_add_u32 s8, s36, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_mov_b32 s40, s7 +; GFX9-DPP-NEXT: s_mov_b32 s41, s6 +; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-DPP-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX9-DPP-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-DPP-NEXT: s_mov_b32 s12, s41 +; GFX9-DPP-NEXT: s_mov_b32 s13, s40 +; GFX9-DPP-NEXT: s_mov_b32 s14, s33 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX9-DPP-NEXT: v_mov_b32_e32 v41, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-DPP-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX9-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX9-DPP-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX9-DPP-NEXT: s_add_u32 s8, s36, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX9-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-DPP-NEXT: s_mov_b32 s12, s41 +; GFX9-DPP-NEXT: s_mov_b32 s13, s40 +; GFX9-DPP-NEXT: s_mov_b32 s14, s33 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX9-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX9-DPP-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX9-DPP-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX9-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB7_1 +; GFX9-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s50, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s51, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX1064-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1064-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1064-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-DPP-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX1064-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v41, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-DPP-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX1064-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1064-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX1064-DPP-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX1064-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX1064-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX1064-DPP-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-DPP-NEXT: s_clause 0x1 +; GFX1064-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX1064-DPP-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX1064-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB7_1 +; GFX1064-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s50, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s51, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX1032-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1032-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1032-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-DPP-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX1032-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-DPP-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v41, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-DPP-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX1032-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1032-DPP-NEXT: s_mov_b32 s44, 0 +; GFX1032-DPP-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX1032-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX1032-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX1032-DPP-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-DPP-NEXT: s_clause 0x1 +; GFX1032-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX1032-DPP-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX1032-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-DPP-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s44 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB7_1 +; GFX1032-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1164-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v41, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: global_load_b64 v[2:3], v41, s[42:43] +; GFX1164-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1164-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-DPP-NEXT: .p2align 6 +; GFX1164-DPP-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1164-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[2:3], off +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[0:1], off offset:8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: scratch_load_b64 v[2:3], off, off +; GFX1164-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[44:45] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB7_1 +; GFX1164-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1132-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1132-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1132-DPP-NEXT: s_mov_b32 s40, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s41, s13 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-DPP-NEXT: s_mov_b32 s12, s13 +; GFX1132-DPP-NEXT: s_mov_b32 s13, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s15 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1132-DPP-NEXT: s_mov_b32 s33, s15 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v41, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: global_load_b64 v[2:3], v41, s[42:43] +; GFX1132-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s44, 0 +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-DPP-NEXT: .p2align 6 +; GFX1132-DPP-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1132-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[2:3], off +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[0:1], off offset:8 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 8 :: v_dual_mov_b32 v1, 0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s42 :: v_dual_mov_b32 v3, s43 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: scratch_load_b64 v[2:3], off, off +; GFX1132-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-DPP-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s44 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB7_1 +; GFX1132-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-DPP-NEXT: s_endpgm + %divValue = call double @div.double.value() + %result = atomicrmw fmax ptr addrspace(1) %ptr, double %divValue syncscope("agent") monotonic, align 4 + ret void +} + +define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_one_as_scope_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fmax_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0 +; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX7LESS-NEXT: s_cbranch_execz .LBB8_3 +; GFX7LESS-NEXT: ; %bb.1: +; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0 +; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s6 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s7 +; GFX7LESS-NEXT: s_mov_b32 s2, -1 +; GFX7LESS-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX7LESS-NEXT: v_max_f64 v[0:1], v[0:1], 4.0 +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, v1 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, v0 +; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3] +; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5] +; GFX7LESS-NEXT: v_mov_b32_e32 v2, v4 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, v5 +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB8_2 +; GFX7LESS-NEXT: .LBB8_3: +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fmax_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX9-NEXT: s_cbranch_execz .LBB8_3 +; GFX9-NEXT: ; %bb.1: +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-NEXT: s_mov_b64 s[2:3], 0 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v2, s4 +; GFX9-NEXT: v_mov_b32_e32 v3, s5 +; GFX9-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX9-NEXT: v_max_f64 v[0:1], v[0:1], 4.0 +; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX9-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX9-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX9-NEXT: s_cbranch_execnz .LBB8_2 +; GFX9-NEXT: .LBB8_3: +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fmax_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX1064-NEXT: s_cbranch_execz .LBB8_3 +; GFX1064-NEXT: ; %bb.1: +; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: v_mov_b32_e32 v2, s2 +; GFX1064-NEXT: v_mov_b32_e32 v3, s3 +; GFX1064-NEXT: s_mov_b64 s[2:3], 0 +; GFX1064-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1064-NEXT: v_max_f64 v[0:1], v[0:1], 4.0 +; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1064-NEXT: v_mov_b32_e32 v3, v1 +; GFX1064-NEXT: v_mov_b32_e32 v2, v0 +; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX1064-NEXT: s_cbranch_execnz .LBB8_2 +; GFX1064-NEXT: .LBB8_3: +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fmax_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1032-NEXT: s_mov_b32 s2, 0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-NEXT: s_and_saveexec_b32 s3, vcc_lo +; GFX1032-NEXT: s_cbranch_execz .LBB8_3 +; GFX1032-NEXT: ; %bb.1: +; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: v_mov_b32_e32 v2, s4 +; GFX1032-NEXT: v_mov_b32_e32 v3, s5 +; GFX1032-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1032-NEXT: v_max_f64 v[0:1], v[0:1], 4.0 +; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1032-NEXT: v_mov_b32_e32 v3, v1 +; GFX1032-NEXT: v_mov_b32_e32 v2, v0 +; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 +; GFX1032-NEXT: s_cbranch_execnz .LBB8_2 +; GFX1032-NEXT: .LBB8_3: +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fmax_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1164-NEXT: s_mov_b64 s[2:3], exec +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-NEXT: s_cbranch_execz .LBB8_3 +; GFX1164-NEXT: ; %bb.1: +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_load_b64 s[2:3], s[0:1], 0x0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: v_mov_b32_e32 v2, s2 +; GFX1164-NEXT: v_mov_b32_e32 v3, s3 +; GFX1164-NEXT: s_mov_b64 s[2:3], 0 +; GFX1164-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1164-NEXT: v_max_f64 v[0:1], v[0:1], 4.0 +; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1164-NEXT: v_mov_b32_e32 v3, v1 +; GFX1164-NEXT: v_mov_b32_e32 v2, v0 +; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3] +; GFX1164-NEXT: s_cbranch_execnz .LBB8_2 +; GFX1164-NEXT: .LBB8_3: +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fmax_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1132-NEXT: s_mov_b32 s2, 0 +; GFX1132-NEXT: s_mov_b32 s3, exec_lo +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-NEXT: s_cbranch_execz .LBB8_3 +; GFX1132-NEXT: ; %bb.1: +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1132-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_load_b64 s[4:5], s[0:1], 0x0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5 +; GFX1132-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1132-NEXT: v_max_f64 v[0:1], v[0:1], 4.0 +; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2 +; GFX1132-NEXT: s_cbranch_execnz .LBB8_2 +; GFX1132-NEXT: .LBB8_3: +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX9-DPP-NEXT: s_cbranch_execz .LBB8_3 +; GFX9-DPP-NEXT: ; %bb.1: +; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s4 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s5 +; GFX9-DPP-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[0:1], 4.0 +; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB8_2 +; GFX9-DPP-NEXT: .LBB8_3: +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX1064-DPP-NEXT: s_cbranch_execz .LBB8_3 +; GFX1064-DPP-NEXT: ; %bb.1: +; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s2 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s3 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX1064-DPP-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1064-DPP-NEXT: v_max_f64 v[0:1], v[0:1], 4.0 +; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB8_2 +; GFX1064-DPP-NEXT: .LBB8_3: +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s2, 0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-DPP-NEXT: s_and_saveexec_b32 s3, vcc_lo +; GFX1032-DPP-NEXT: s_cbranch_execz .LBB8_3 +; GFX1032-DPP-NEXT: ; %bb.1: +; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s4 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s5 +; GFX1032-DPP-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1032-DPP-NEXT: v_max_f64 v[0:1], v[0:1], 4.0 +; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB8_2 +; GFX1032-DPP-NEXT: .LBB8_3: +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-DPP-NEXT: s_cbranch_execz .LBB8_3 +; GFX1164-DPP-NEXT: ; %bb.1: +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s2 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s3 +; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX1164-DPP-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], 4.0 +; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB8_2 +; GFX1164-DPP-NEXT: .LBB8_3: +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s2, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s3, exec_lo +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-DPP-NEXT: s_cbranch_execz .LBB8_3 +; GFX1132-DPP-NEXT: ; %bb.1: +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5 +; GFX1132-DPP-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[0:1], 4.0 +; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB8_2 +; GFX1132-DPP-NEXT: .LBB8_3: +; GFX1132-DPP-NEXT: s_endpgm + %result = atomicrmw fmax ptr addrspace(1) %ptr, double 4.0 syncscope("one-as") monotonic + ret void +} + +define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_as_scope_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fmax_double_uni_address_div_value_one_as_scope_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_mov_b32 s32, 0 +; GFX7LESS-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s42, -1 +; GFX7LESS-NEXT: s_mov_b32 s43, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s40, s40, s9 +; GFX7LESS-NEXT: s_addc_u32 s41, s41, 0 +; GFX7LESS-NEXT: s_mov_b32 s14, s8 +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX7LESS-NEXT: s_load_dwordx2 s[36:37], s[2:3], 0x9 +; GFX7LESS-NEXT: s_mov_b32 s39, 0xf000 +; GFX7LESS-NEXT: s_mov_b32 s38, -1 +; GFX7LESS-NEXT: s_add_u32 s8, s2, 44 +; GFX7LESS-NEXT: s_addc_u32 s9, s3, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[2:3] +; GFX7LESS-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v31, v0, v2 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX7LESS-NEXT: s_mov_b32 s12, s6 +; GFX7LESS-NEXT: s_mov_b32 s13, s7 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX7LESS-NEXT: buffer_load_dwordx2 v[2:3], off, s[36:39], 0 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], 0 +; GFX7LESS-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX7LESS-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX7LESS-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3 +; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0 +; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[36:39], 0 glc +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3] +; GFX7LESS-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7 +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB9_1 +; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fmax_double_uni_address_div_value_one_as_scope_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s38, -1 +; GFX9-NEXT: s_mov_b32 s39, 0xe00000 +; GFX9-NEXT: s_add_u32 s36, s36, s9 +; GFX9-NEXT: s_addc_u32 s37, s37, 0 +; GFX9-NEXT: s_mov_b32 s14, s8 +; GFX9-NEXT: s_add_u32 s8, s2, 44 +; GFX9-NEXT: s_addc_u32 s9, s3, 0 +; GFX9-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX9-NEXT: s_getpc_b64 s[2:3] +; GFX9-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX9-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX9-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX9-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX9-NEXT: s_mov_b32 s12, s6 +; GFX9-NEXT: s_mov_b32 s13, s7 +; GFX9-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX9-NEXT: s_mov_b32 s32, 0 +; GFX9-NEXT: v_mov_b32_e32 v40, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX9-NEXT: global_load_dwordx2 v[2:3], v40, s[34:35] +; GFX9-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX9-NEXT: s_mov_b64 s[0:1], 0 +; GFX9-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX9-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5] +; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v40, v[0:3], s[34:35] glc +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX9-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX9-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX9-NEXT: s_cbranch_execnz .LBB9_1 +; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fmax_double_uni_address_div_value_one_as_scope_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s38, -1 +; GFX1064-NEXT: s_mov_b32 s39, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s36, s36, s9 +; GFX1064-NEXT: s_addc_u32 s37, s37, 0 +; GFX1064-NEXT: s_mov_b32 s14, s8 +; GFX1064-NEXT: s_add_u32 s8, s2, 44 +; GFX1064-NEXT: s_addc_u32 s9, s3, 0 +; GFX1064-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1064-NEXT: s_getpc_b64 s[4:5] +; GFX1064-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1064-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1064-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1064-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1064-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1064-NEXT: s_mov_b32 s12, s6 +; GFX1064-NEXT: s_mov_b32 s13, s7 +; GFX1064-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1064-NEXT: s_mov_b32 s32, 0 +; GFX1064-NEXT: v_mov_b32_e32 v40, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1064-NEXT: global_load_dwordx2 v[2:3], v40, s[34:35] +; GFX1064-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX1064-NEXT: s_mov_b64 s[0:1], 0 +; GFX1064-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1064-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5] +; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v40, v[0:3], s[34:35] glc +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1064-NEXT: v_mov_b32_e32 v3, v1 +; GFX1064-NEXT: v_mov_b32_e32 v2, v0 +; GFX1064-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX1064-NEXT: s_cbranch_execnz .LBB9_1 +; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fmax_double_uni_address_div_value_one_as_scope_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s38, -1 +; GFX1032-NEXT: s_mov_b32 s39, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s36, s36, s9 +; GFX1032-NEXT: s_addc_u32 s37, s37, 0 +; GFX1032-NEXT: s_mov_b32 s14, s8 +; GFX1032-NEXT: s_add_u32 s8, s2, 44 +; GFX1032-NEXT: s_addc_u32 s9, s3, 0 +; GFX1032-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1032-NEXT: s_getpc_b64 s[4:5] +; GFX1032-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1032-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1032-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1032-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1032-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1032-NEXT: s_mov_b32 s12, s6 +; GFX1032-NEXT: s_mov_b32 s13, s7 +; GFX1032-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1032-NEXT: s_mov_b32 s32, 0 +; GFX1032-NEXT: v_mov_b32_e32 v40, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1032-NEXT: global_load_dwordx2 v[2:3], v40, s[34:35] +; GFX1032-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX1032-NEXT: s_mov_b32 s0, 0 +; GFX1032-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1032-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5] +; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v40, v[0:3], s[34:35] glc +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1032-NEXT: v_mov_b32_e32 v3, v1 +; GFX1032-NEXT: v_mov_b32_e32 v2, v0 +; GFX1032-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 +; GFX1032-NEXT: s_cbranch_execnz .LBB9_1 +; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fmax_double_uni_address_div_value_one_as_scope_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b32 s14, s8 +; GFX1164-NEXT: s_add_u32 s8, s2, 44 +; GFX1164-NEXT: s_addc_u32 s9, s3, 0 +; GFX1164-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1164-NEXT: s_getpc_b64 s[4:5] +; GFX1164-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1164-NEXT: s_load_b64 s[16:17], s[4:5], 0x0 +; GFX1164-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1164-NEXT: s_mov_b32 s12, s6 +; GFX1164-NEXT: s_mov_b32 s13, s7 +; GFX1164-NEXT: s_mov_b32 s32, 0 +; GFX1164-NEXT: v_mov_b32_e32 v40, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1164-NEXT: global_load_b64 v[2:3], v40, s[34:35] +; GFX1164-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX1164-NEXT: s_mov_b64 s[0:1], 0 +; GFX1164-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5] +; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v40, v[0:3], s[34:35] glc +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1164-NEXT: v_mov_b32_e32 v3, v1 +; GFX1164-NEXT: v_mov_b32_e32 v2, v0 +; GFX1164-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[0:1] +; GFX1164-NEXT: s_cbranch_execnz .LBB9_1 +; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fmax_double_uni_address_div_value_one_as_scope_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_add_u32 s8, s2, 44 +; GFX1132-NEXT: s_addc_u32 s9, s3, 0 +; GFX1132-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1132-NEXT: s_getpc_b64 s[4:5] +; GFX1132-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1132-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 +; GFX1132-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1132-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v31, v0 +; GFX1132-NEXT: s_mov_b32 s12, s13 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1132-NEXT: s_mov_b32 s13, s14 +; GFX1132-NEXT: s_mov_b32 s14, s15 +; GFX1132-NEXT: s_mov_b32 s32, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1132-NEXT: global_load_b64 v[2:3], v40, s[34:35] +; GFX1132-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX1132-NEXT: s_mov_b32 s0, 0 +; GFX1132-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5] +; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v40, v[0:3], s[34:35] glc +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX1132-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX1132-NEXT: s_cbranch_execnz .LBB9_1 +; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_one_as_scope_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s38, -1 +; GFX9-DPP-NEXT: s_mov_b32 s39, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX9-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX9-DPP-NEXT: s_mov_b32 s14, s8 +; GFX9-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX9-DPP-NEXT: s_getpc_b64 s[2:3] +; GFX9-DPP-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX9-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX9-DPP-NEXT: s_mov_b32 s12, s6 +; GFX9-DPP-NEXT: s_mov_b32 s13, s7 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX9-DPP-NEXT: s_mov_b32 s32, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX9-DPP-NEXT: global_load_dwordx2 v[2:3], v40, s[34:35] +; GFX9-DPP-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX9-DPP-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5] +; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v40, v[0:3], s[34:35] glc +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB9_1 +; GFX9-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_one_as_scope_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s38, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s39, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX1064-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1064-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1064-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1064-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1064-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1064-DPP-NEXT: global_load_dwordx2 v[2:3], v40, s[34:35] +; GFX1064-DPP-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX1064-DPP-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1064-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5] +; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v40, v[0:3], s[34:35] glc +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB9_1 +; GFX1064-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_one_as_scope_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s38, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s39, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX1032-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1032-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1032-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1032-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1032-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1032-DPP-NEXT: global_load_dwordx2 v[2:3], v40, s[34:35] +; GFX1032-DPP-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX1032-DPP-NEXT: s_mov_b32 s0, 0 +; GFX1032-DPP-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1032-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5] +; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v40, v[0:3], s[34:35] glc +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1032-DPP-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB9_1 +; GFX1032-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_one_as_scope_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1164-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1164-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1164-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: s_load_b64 s[16:17], s[4:5], 0x0 +; GFX1164-DPP-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1164-DPP-NEXT: global_load_b64 v[2:3], v40, s[34:35] +; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX1164-DPP-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5] +; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v40, v[0:3], s[34:35] glc +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[0:1] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB9_1 +; GFX1164-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_one_as_scope_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1132-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1132-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 +; GFX1132-DPP-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v31, v0 +; GFX1132-DPP-NEXT: s_mov_b32 s12, s13 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s13, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s15 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1132-DPP-NEXT: global_load_b64 v[2:3], v40, s[34:35] +; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s0, 0 +; GFX1132-DPP-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5] +; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v40, v[0:3], s[34:35] glc +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX1132-DPP-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB9_1 +; GFX1132-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-DPP-NEXT: s_endpgm + %divValue = call double @div.double.value() + %result = atomicrmw fmax ptr addrspace(1) %ptr, double %divValue syncscope("one-as") monotonic + ret void +} + +define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defalut_scope_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fmax_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_movk_i32 s32, 0x800 +; GFX7LESS-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s42, -1 +; GFX7LESS-NEXT: s_mov_b32 s43, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s40, s40, s3 +; GFX7LESS-NEXT: s_addc_u32 s41, s41, 0 +; GFX7LESS-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX7LESS-NEXT: v_mov_b32_e32 v40, v0 +; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0 +; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX7LESS-NEXT: s_cbranch_execz .LBB10_3 +; GFX7LESS-NEXT: ; %bb.1: +; GFX7LESS-NEXT: s_mov_b32 s33, s2 +; GFX7LESS-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x9 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX7LESS-NEXT: s_mov_b64 s[38:39], 0 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, s0 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, s1 +; GFX7LESS-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_max_f64 v[2:3], v[0:1], v[0:1] +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[40:43], 0 offset:4 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[40:43], 0 +; GFX7LESS-NEXT: s_add_u32 s8, s34, 44 +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_max_f64 v[0:1], v[2:3], 4.0 +; GFX7LESS-NEXT: s_addc_u32 s9, s35, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[40:43], 0 offset:12 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[40:43], 0 offset:8 +; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, 0 +; GFX7LESS-NEXT: s_mov_b32 s12, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v40 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s36 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s37 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX7LESS-NEXT: v_and_b32_e32 v2, 1, v0 +; GFX7LESS-NEXT: buffer_load_dword v0, off, s[40:43], 0 +; GFX7LESS-NEXT: buffer_load_dword v1, off, s[40:43], 0 offset:4 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2 +; GFX7LESS-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB10_2 +; GFX7LESS-NEXT: .LBB10_3: +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fmax_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s42, -1 +; GFX9-NEXT: s_mov_b32 s43, 0xe00000 +; GFX9-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX9-NEXT: s_add_u32 s40, s40, s3 +; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-NEXT: s_addc_u32 s41, s41, 0 +; GFX9-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-NEXT: s_movk_i32 s32, 0x800 +; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX9-NEXT: s_cbranch_execz .LBB10_3 +; GFX9-NEXT: ; %bb.1: +; GFX9-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX9-NEXT: s_mov_b32 s33, s2 +; GFX9-NEXT: s_mov_b64 s[38:39], 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX9-NEXT: s_add_u32 s8, s34, 44 +; GFX9-NEXT: s_addc_u32 s9, s35, 0 +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX9-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX9-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX9-NEXT: s_mov_b32 s12, s33 +; GFX9-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX9-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX9-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: v_mov_b32_e32 v2, s36 +; GFX9-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX9-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX9-NEXT: v_mov_b32_e32 v3, s37 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX9-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX9-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX9-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX9-NEXT: s_cbranch_execnz .LBB10_2 +; GFX9-NEXT: .LBB10_3: +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fmax_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1064-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s42, -1 +; GFX1064-NEXT: s_mov_b32 s43, 0x31e16000 +; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1064-NEXT: s_add_u32 s40, s40, s3 +; GFX1064-NEXT: s_addc_u32 s41, s41, 0 +; GFX1064-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1064-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX1064-NEXT: s_cbranch_execz .LBB10_3 +; GFX1064-NEXT: ; %bb.1: +; GFX1064-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1064-NEXT: s_mov_b32 s33, s2 +; GFX1064-NEXT: s_mov_b64 s[38:39], 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: v_mov_b32_e32 v2, s1 +; GFX1064-NEXT: v_mov_b32_e32 v1, s0 +; GFX1064-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1064-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1064-NEXT: s_mov_b32 s12, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1064-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX1064-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1064-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1064-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1064-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-NEXT: v_mov_b32_e32 v2, s36 +; GFX1064-NEXT: v_mov_b32_e32 v3, s37 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1064-NEXT: s_clause 0x1 +; GFX1064-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1064-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX1064-NEXT: s_cbranch_execnz .LBB10_2 +; GFX1064-NEXT: .LBB10_3: +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fmax_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1032-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s42, -1 +; GFX1032-NEXT: s_mov_b32 s43, 0x31c16000 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-NEXT: s_add_u32 s40, s40, s3 +; GFX1032-NEXT: s_addc_u32 s41, s41, 0 +; GFX1032-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1032-NEXT: s_mov_b32 s38, 0 +; GFX1032-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo +; GFX1032-NEXT: s_cbranch_execz .LBB10_3 +; GFX1032-NEXT: ; %bb.1: +; GFX1032-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1032-NEXT: s_mov_b32 s33, s2 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: v_mov_b32_e32 v2, s1 +; GFX1032-NEXT: v_mov_b32_e32 v1, s0 +; GFX1032-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1032-NEXT: s_mov_b32 s12, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1032-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX1032-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1032-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1032-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1032-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-NEXT: v_mov_b32_e32 v2, s36 +; GFX1032-NEXT: v_mov_b32_e32 v3, s37 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1032-NEXT: s_clause 0x1 +; GFX1032-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1032-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s38 +; GFX1032-NEXT: s_cbranch_execnz .LBB10_2 +; GFX1032-NEXT: .LBB10_3: +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fmax_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1164-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1164-NEXT: s_mov_b32 s32, 32 +; GFX1164-NEXT: s_mov_b64 s[0:1], exec +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-NEXT: s_cbranch_execz .LBB10_3 +; GFX1164-NEXT: ; %bb.1: +; GFX1164-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1164-NEXT: s_mov_b32 s33, s2 +; GFX1164-NEXT: s_mov_b64 s[38:39], 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: v_mov_b32_e32 v2, s1 +; GFX1164-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-NEXT: .p2align 6 +; GFX1164-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-NEXT: s_mov_b32 s12, s33 +; GFX1164-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-NEXT: v_mov_b32_e32 v2, s36 +; GFX1164-NEXT: v_mov_b32_e32 v3, s37 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[38:39] +; GFX1164-NEXT: s_cbranch_execnz .LBB10_2 +; GFX1164-NEXT: .LBB10_3: +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fmax_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: v_mov_b32_e32 v40, v0 +; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1132-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1132-NEXT: s_mov_b32 s38, 0 +; GFX1132-NEXT: s_mov_b32 s32, 32 +; GFX1132-NEXT: s_mov_b32 s0, exec_lo +; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-NEXT: s_cbranch_execz .LBB10_3 +; GFX1132-NEXT: ; %bb.1: +; GFX1132-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1132-NEXT: s_mov_b32 s33, s15 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0 +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-NEXT: .p2align 6 +; GFX1132-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-NEXT: v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-NEXT: s_mov_b32 s12, s33 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX1132-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s36 +; GFX1132-NEXT: v_dual_mov_b32 v3, s37 :: v_dual_mov_b32 v4, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38 +; GFX1132-NEXT: s_cbranch_execnz .LBB10_2 +; GFX1132-NEXT: .LBB10_3: +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s42, -1 +; GFX9-DPP-NEXT: s_mov_b32 s43, 0xe00000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX9-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX9-DPP-NEXT: s_cbranch_execz .LBB10_3 +; GFX9-DPP-NEXT: ; %bb.1: +; GFX9-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX9-DPP-NEXT: s_mov_b32 s33, s2 +; GFX9-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-DPP-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX9-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX9-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX9-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX9-DPP-NEXT: s_mov_b32 s12, s33 +; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX9-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX9-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX9-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX9-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB10_2 +; GFX9-DPP-NEXT: .LBB10_3: +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s42, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s43, 0x31e16000 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1064-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX1064-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1064-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX1064-DPP-NEXT: s_cbranch_execz .LBB10_3 +; GFX1064-DPP-NEXT: ; %bb.1: +; GFX1064-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1064-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1064-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1064-DPP-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX1064-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1064-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1064-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1064-DPP-NEXT: s_clause 0x1 +; GFX1064-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1064-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB10_2 +; GFX1064-DPP-NEXT: .LBB10_3: +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s42, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s43, 0x31c16000 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX1032-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1032-DPP-NEXT: s_mov_b32 s38, 0 +; GFX1032-DPP-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo +; GFX1032-DPP-NEXT: s_cbranch_execz .LBB10_3 +; GFX1032-DPP-NEXT: ; %bb.1: +; GFX1032-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1032-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1032-DPP-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX1032-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1032-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1032-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1032-DPP-NEXT: s_clause 0x1 +; GFX1032-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1032-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-DPP-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s38 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB10_2 +; GFX1032-DPP-NEXT: .LBB10_3: +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1164-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-DPP-NEXT: s_cbranch_execz .LBB10_3 +; GFX1164-DPP-NEXT: ; %bb.1: +; GFX1164-DPP-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1164-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1164-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-DPP-NEXT: .p2align 6 +; GFX1164-DPP-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1164-DPP-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[38:39] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB10_2 +; GFX1164-DPP-NEXT: .LBB10_3: +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s38, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-DPP-NEXT: s_cbranch_execz .LBB10_3 +; GFX1132-DPP-NEXT: ; %bb.1: +; GFX1132-DPP-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1132-DPP-NEXT: s_mov_b32 s33, s15 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0 +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-DPP-NEXT: .p2align 6 +; GFX1132-DPP-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX1132-DPP-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s36 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, s37 :: v_dual_mov_b32 v4, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-DPP-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB10_2 +; GFX1132-DPP-NEXT: .LBB10_3: +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-DPP-NEXT: s_endpgm + %result = atomicrmw fmax ptr addrspace(1) %ptr, double 4.0 monotonic, align 4 + ret void +} + +define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defalut_scope_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fmax_double_uni_address_div_value_defalut_scope_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_movk_i32 s32, 0x800 +; GFX7LESS-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s50, -1 +; GFX7LESS-NEXT: s_mov_b32 s51, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s48, s48, s9 +; GFX7LESS-NEXT: s_addc_u32 s49, s49, 0 +; GFX7LESS-NEXT: s_mov_b32 s33, s8 +; GFX7LESS-NEXT: s_mov_b32 s40, s7 +; GFX7LESS-NEXT: s_mov_b32 s41, s6 +; GFX7LESS-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX7LESS-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX7LESS-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX7LESS-NEXT: s_load_dwordx2 s[44:45], s[2:3], 0x9 +; GFX7LESS-NEXT: s_mov_b32 s47, 0xf000 +; GFX7LESS-NEXT: s_mov_b32 s46, -1 +; GFX7LESS-NEXT: s_add_u32 s8, s36, 44 +; GFX7LESS-NEXT: s_addc_u32 s9, s37, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v40, v0, v2 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX7LESS-NEXT: s_mov_b32 s12, s41 +; GFX7LESS-NEXT: s_mov_b32 s13, s40 +; GFX7LESS-NEXT: s_mov_b32 s14, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v40 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX7LESS-NEXT: buffer_load_dwordx2 v[2:3], off, s[44:47], 0 +; GFX7LESS-NEXT: s_mov_b64 s[42:43], 0 +; GFX7LESS-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX7LESS-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX7LESS-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX7LESS-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX7LESS-NEXT: s_add_u32 s8, s36, 44 +; GFX7LESS-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX7LESS-NEXT: s_addc_u32 s9, s37, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, 0 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX7LESS-NEXT: s_mov_b32 s12, s41 +; GFX7LESS-NEXT: s_mov_b32 s13, s40 +; GFX7LESS-NEXT: s_mov_b32 s14, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v40 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s44 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s45 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX7LESS-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX7LESS-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX7LESS-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX7LESS-NEXT: s_or_b64 s[42:43], vcc, s[42:43] +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[42:43] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB11_1 +; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fmax_double_uni_address_div_value_defalut_scope_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s50, -1 +; GFX9-NEXT: s_mov_b32 s51, 0xe00000 +; GFX9-NEXT: s_add_u32 s48, s48, s9 +; GFX9-NEXT: s_addc_u32 s49, s49, 0 +; GFX9-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX9-NEXT: s_mov_b32 s33, s8 +; GFX9-NEXT: s_add_u32 s8, s36, 44 +; GFX9-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX9-NEXT: s_mov_b32 s40, s7 +; GFX9-NEXT: s_mov_b32 s41, s6 +; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX9-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX9-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-NEXT: s_mov_b32 s12, s41 +; GFX9-NEXT: s_mov_b32 s13, s40 +; GFX9-NEXT: s_mov_b32 s14, s33 +; GFX9-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-NEXT: s_movk_i32 s32, 0x800 +; GFX9-NEXT: v_mov_b32_e32 v41, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX9-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX9-NEXT: s_mov_b64 s[44:45], 0 +; GFX9-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX9-NEXT: s_add_u32 s8, s36, 44 +; GFX9-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX9-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX9-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX9-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-NEXT: s_mov_b32 s12, s41 +; GFX9-NEXT: s_mov_b32 s13, s40 +; GFX9-NEXT: s_mov_b32 s14, s33 +; GFX9-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-NEXT: v_mov_b32_e32 v2, s42 +; GFX9-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX9-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX9-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: v_mov_b32_e32 v3, s43 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX9-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX9-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX9-NEXT: s_cbranch_execnz .LBB11_1 +; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fmax_double_uni_address_div_value_defalut_scope_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s50, -1 +; GFX1064-NEXT: s_mov_b32 s51, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s48, s48, s9 +; GFX1064-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1064-NEXT: s_addc_u32 s49, s49, 0 +; GFX1064-NEXT: s_mov_b32 s33, s8 +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1064-NEXT: s_mov_b32 s40, s7 +; GFX1064-NEXT: s_mov_b32 s41, s6 +; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1064-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1064-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX1064-NEXT: s_mov_b32 s12, s41 +; GFX1064-NEXT: s_mov_b32 s13, s40 +; GFX1064-NEXT: s_mov_b32 s14, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-NEXT: v_mov_b32_e32 v41, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX1064-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1064-NEXT: s_mov_b64 s[44:45], 0 +; GFX1064-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX1064-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-NEXT: v_mov_b32_e32 v2, s42 +; GFX1064-NEXT: v_mov_b32_e32 v3, s43 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-NEXT: s_mov_b32 s12, s41 +; GFX1064-NEXT: s_mov_b32 s13, s40 +; GFX1064-NEXT: s_mov_b32 s14, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX1064-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX1064-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX1064-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-NEXT: s_clause 0x1 +; GFX1064-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX1064-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX1064-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX1064-NEXT: s_cbranch_execnz .LBB11_1 +; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fmax_double_uni_address_div_value_defalut_scope_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s50, -1 +; GFX1032-NEXT: s_mov_b32 s51, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s48, s48, s9 +; GFX1032-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1032-NEXT: s_addc_u32 s49, s49, 0 +; GFX1032-NEXT: s_mov_b32 s33, s8 +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1032-NEXT: s_mov_b32 s40, s7 +; GFX1032-NEXT: s_mov_b32 s41, s6 +; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1032-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1032-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX1032-NEXT: s_mov_b32 s12, s41 +; GFX1032-NEXT: s_mov_b32 s13, s40 +; GFX1032-NEXT: s_mov_b32 s14, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-NEXT: v_mov_b32_e32 v41, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX1032-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1032-NEXT: s_mov_b32 s44, 0 +; GFX1032-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX1032-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-NEXT: v_mov_b32_e32 v2, s42 +; GFX1032-NEXT: v_mov_b32_e32 v3, s43 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-NEXT: s_mov_b32 s12, s41 +; GFX1032-NEXT: s_mov_b32 s13, s40 +; GFX1032-NEXT: s_mov_b32 s14, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX1032-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX1032-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX1032-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-NEXT: s_clause 0x1 +; GFX1032-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX1032-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX1032-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s44 +; GFX1032-NEXT: s_cbranch_execnz .LBB11_1 +; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fmax_double_uni_address_div_value_defalut_scope_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1164-NEXT: s_mov_b32 s33, s8 +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1164-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-NEXT: s_mov_b32 s12, s6 +; GFX1164-NEXT: s_mov_b32 s13, s7 +; GFX1164-NEXT: s_mov_b32 s14, s33 +; GFX1164-NEXT: s_mov_b32 s32, 32 +; GFX1164-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-NEXT: s_mov_b32 s40, s7 +; GFX1164-NEXT: s_mov_b32 s41, s6 +; GFX1164-NEXT: v_mov_b32_e32 v41, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: global_load_b64 v[2:3], v41, s[42:43] +; GFX1164-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1164-NEXT: s_mov_b64 s[44:45], 0 +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-NEXT: .p2align 6 +; GFX1164-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-NEXT: s_mov_b32 s12, s41 +; GFX1164-NEXT: s_mov_b32 s13, s40 +; GFX1164-NEXT: s_mov_b32 s14, s33 +; GFX1164-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b64 off, v[2:3], off +; GFX1164-NEXT: scratch_store_b64 off, v[0:1], off offset:8 +; GFX1164-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-NEXT: v_mov_b32_e32 v2, s42 +; GFX1164-NEXT: v_mov_b32_e32 v3, s43 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: scratch_load_b64 v[2:3], off, off +; GFX1164-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[44:45] +; GFX1164-NEXT: s_cbranch_execnz .LBB11_1 +; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fmax_double_uni_address_div_value_defalut_scope_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1132-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1132-NEXT: v_mov_b32_e32 v31, v0 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1132-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1132-NEXT: s_mov_b32 s40, s14 +; GFX1132-NEXT: s_mov_b32 s41, s13 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-NEXT: s_mov_b32 s12, s13 +; GFX1132-NEXT: s_mov_b32 s13, s14 +; GFX1132-NEXT: s_mov_b32 s14, s15 +; GFX1132-NEXT: s_mov_b32 s32, 32 +; GFX1132-NEXT: s_mov_b32 s33, s15 +; GFX1132-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v41, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: global_load_b64 v[2:3], v41, s[42:43] +; GFX1132-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1132-NEXT: s_mov_b32 s44, 0 +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-NEXT: .p2align 6 +; GFX1132-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-NEXT: v_mov_b32_e32 v31, v40 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-NEXT: s_mov_b32 s12, s41 +; GFX1132-NEXT: s_mov_b32 s13, s40 +; GFX1132-NEXT: s_mov_b32 s14, s33 +; GFX1132-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b64 off, v[2:3], off +; GFX1132-NEXT: scratch_store_b64 off, v[0:1], off offset:8 +; GFX1132-NEXT: v_dual_mov_b32 v0, 8 :: v_dual_mov_b32 v1, 0 +; GFX1132-NEXT: v_dual_mov_b32 v2, s42 :: v_dual_mov_b32 v3, s43 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: scratch_load_b64 v[2:3], off, off +; GFX1132-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s44 +; GFX1132-NEXT: s_cbranch_execnz .LBB11_1 +; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_defalut_scope_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s50, -1 +; GFX9-DPP-NEXT: s_mov_b32 s51, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX9-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX9-DPP-NEXT: s_mov_b32 s33, s8 +; GFX9-DPP-NEXT: s_add_u32 s8, s36, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_mov_b32 s40, s7 +; GFX9-DPP-NEXT: s_mov_b32 s41, s6 +; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-DPP-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX9-DPP-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-DPP-NEXT: s_mov_b32 s12, s41 +; GFX9-DPP-NEXT: s_mov_b32 s13, s40 +; GFX9-DPP-NEXT: s_mov_b32 s14, s33 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX9-DPP-NEXT: v_mov_b32_e32 v41, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-DPP-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX9-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX9-DPP-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX9-DPP-NEXT: s_add_u32 s8, s36, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX9-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-DPP-NEXT: s_mov_b32 s12, s41 +; GFX9-DPP-NEXT: s_mov_b32 s13, s40 +; GFX9-DPP-NEXT: s_mov_b32 s14, s33 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX9-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX9-DPP-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX9-DPP-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX9-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB11_1 +; GFX9-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_defalut_scope_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s50, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s51, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX1064-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1064-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1064-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-DPP-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX1064-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v41, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-DPP-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX1064-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1064-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX1064-DPP-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX1064-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX1064-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX1064-DPP-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-DPP-NEXT: s_clause 0x1 +; GFX1064-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX1064-DPP-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX1064-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB11_1 +; GFX1064-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_defalut_scope_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s50, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s51, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX1032-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1032-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1032-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-DPP-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX1032-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-DPP-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v41, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-DPP-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX1032-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1032-DPP-NEXT: s_mov_b32 s44, 0 +; GFX1032-DPP-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX1032-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX1032-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX1032-DPP-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-DPP-NEXT: s_clause 0x1 +; GFX1032-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX1032-DPP-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX1032-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-DPP-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s44 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB11_1 +; GFX1032-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_defalut_scope_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1164-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v41, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: global_load_b64 v[2:3], v41, s[42:43] +; GFX1164-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1164-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-DPP-NEXT: .p2align 6 +; GFX1164-DPP-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1164-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[2:3], off +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[0:1], off offset:8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: scratch_load_b64 v[2:3], off, off +; GFX1164-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[44:45] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB11_1 +; GFX1164-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_defalut_scope_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1132-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1132-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1132-DPP-NEXT: s_mov_b32 s40, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s41, s13 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-DPP-NEXT: s_mov_b32 s12, s13 +; GFX1132-DPP-NEXT: s_mov_b32 s13, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s15 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1132-DPP-NEXT: s_mov_b32 s33, s15 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v41, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: global_load_b64 v[2:3], v41, s[42:43] +; GFX1132-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s44, 0 +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-DPP-NEXT: .p2align 6 +; GFX1132-DPP-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1132-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[2:3], off +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[0:1], off offset:8 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 8 :: v_dual_mov_b32 v1, 0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s42 :: v_dual_mov_b32 v3, s43 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: scratch_load_b64 v[2:3], off, off +; GFX1132-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-DPP-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s44 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB11_1 +; GFX1132-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-DPP-NEXT: s_endpgm + %divValue = call double @div.double.value() + %result = atomicrmw fmax ptr addrspace(1) %ptr, double %divValue monotonic, align 4 + ret void +} + attributes #0 = { "denormal-fp-math-f32"="preserve-sign,preserve-sign" "amdgpu-unsafe-fp-atomics"="true" } !llvm.module.flags = !{!0} diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll index 314c52a71d93..49d415c9eed7 100644 --- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll +++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll @@ -13,6 +13,7 @@ ; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,-wavefrontsize64 -amdgpu-atomic-optimizer-strategy=DPP -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX1132-DPP %s declare float @div.float.value() +declare float @div.double.value() define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 { ; GFX7LESS-LABEL: global_atomic_fmin_uni_address_uni_value_agent_scope_unsafe: @@ -3550,6 +3551,3965 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_defalut_scop ret void } +define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_movk_i32 s32, 0x800 +; GFX7LESS-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s42, -1 +; GFX7LESS-NEXT: s_mov_b32 s43, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s40, s40, s3 +; GFX7LESS-NEXT: s_addc_u32 s41, s41, 0 +; GFX7LESS-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX7LESS-NEXT: v_mov_b32_e32 v40, v0 +; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0 +; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX7LESS-NEXT: s_cbranch_execz .LBB6_3 +; GFX7LESS-NEXT: ; %bb.1: +; GFX7LESS-NEXT: s_mov_b32 s33, s2 +; GFX7LESS-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x9 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX7LESS-NEXT: s_mov_b64 s[38:39], 0 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, s0 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, s1 +; GFX7LESS-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_max_f64 v[2:3], v[0:1], v[0:1] +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[40:43], 0 offset:4 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[40:43], 0 +; GFX7LESS-NEXT: s_add_u32 s8, s34, 44 +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_min_f64 v[0:1], v[2:3], 4.0 +; GFX7LESS-NEXT: s_addc_u32 s9, s35, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[40:43], 0 offset:12 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[40:43], 0 offset:8 +; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, 0 +; GFX7LESS-NEXT: s_mov_b32 s12, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v40 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s36 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s37 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX7LESS-NEXT: v_and_b32_e32 v2, 1, v0 +; GFX7LESS-NEXT: buffer_load_dword v0, off, s[40:43], 0 +; GFX7LESS-NEXT: buffer_load_dword v1, off, s[40:43], 0 offset:4 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2 +; GFX7LESS-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB6_2 +; GFX7LESS-NEXT: .LBB6_3: +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s42, -1 +; GFX9-NEXT: s_mov_b32 s43, 0xe00000 +; GFX9-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX9-NEXT: s_add_u32 s40, s40, s3 +; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-NEXT: s_addc_u32 s41, s41, 0 +; GFX9-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-NEXT: s_movk_i32 s32, 0x800 +; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX9-NEXT: s_cbranch_execz .LBB6_3 +; GFX9-NEXT: ; %bb.1: +; GFX9-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX9-NEXT: s_mov_b32 s33, s2 +; GFX9-NEXT: s_mov_b64 s[38:39], 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX9-NEXT: s_add_u32 s8, s34, 44 +; GFX9-NEXT: s_addc_u32 s9, s35, 0 +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX9-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX9-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX9-NEXT: s_mov_b32 s12, s33 +; GFX9-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX9-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX9-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: v_mov_b32_e32 v2, s36 +; GFX9-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX9-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX9-NEXT: v_mov_b32_e32 v3, s37 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX9-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX9-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX9-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX9-NEXT: s_cbranch_execnz .LBB6_2 +; GFX9-NEXT: .LBB6_3: +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1064-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s42, -1 +; GFX1064-NEXT: s_mov_b32 s43, 0x31e16000 +; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1064-NEXT: s_add_u32 s40, s40, s3 +; GFX1064-NEXT: s_addc_u32 s41, s41, 0 +; GFX1064-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1064-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX1064-NEXT: s_cbranch_execz .LBB6_3 +; GFX1064-NEXT: ; %bb.1: +; GFX1064-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1064-NEXT: s_mov_b32 s33, s2 +; GFX1064-NEXT: s_mov_b64 s[38:39], 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: v_mov_b32_e32 v2, s1 +; GFX1064-NEXT: v_mov_b32_e32 v1, s0 +; GFX1064-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1064-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1064-NEXT: s_mov_b32 s12, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1064-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX1064-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1064-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1064-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1064-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-NEXT: v_mov_b32_e32 v2, s36 +; GFX1064-NEXT: v_mov_b32_e32 v3, s37 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1064-NEXT: s_clause 0x1 +; GFX1064-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1064-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX1064-NEXT: s_cbranch_execnz .LBB6_2 +; GFX1064-NEXT: .LBB6_3: +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1032-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s42, -1 +; GFX1032-NEXT: s_mov_b32 s43, 0x31c16000 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-NEXT: s_add_u32 s40, s40, s3 +; GFX1032-NEXT: s_addc_u32 s41, s41, 0 +; GFX1032-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1032-NEXT: s_mov_b32 s38, 0 +; GFX1032-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo +; GFX1032-NEXT: s_cbranch_execz .LBB6_3 +; GFX1032-NEXT: ; %bb.1: +; GFX1032-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1032-NEXT: s_mov_b32 s33, s2 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: v_mov_b32_e32 v2, s1 +; GFX1032-NEXT: v_mov_b32_e32 v1, s0 +; GFX1032-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1032-NEXT: s_mov_b32 s12, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1032-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX1032-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1032-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1032-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1032-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-NEXT: v_mov_b32_e32 v2, s36 +; GFX1032-NEXT: v_mov_b32_e32 v3, s37 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1032-NEXT: s_clause 0x1 +; GFX1032-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1032-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s38 +; GFX1032-NEXT: s_cbranch_execnz .LBB6_2 +; GFX1032-NEXT: .LBB6_3: +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1164-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1164-NEXT: s_mov_b32 s32, 32 +; GFX1164-NEXT: s_mov_b64 s[0:1], exec +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-NEXT: s_cbranch_execz .LBB6_3 +; GFX1164-NEXT: ; %bb.1: +; GFX1164-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1164-NEXT: s_mov_b32 s33, s2 +; GFX1164-NEXT: s_mov_b64 s[38:39], 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: v_mov_b32_e32 v2, s1 +; GFX1164-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-NEXT: .p2align 6 +; GFX1164-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-NEXT: s_mov_b32 s12, s33 +; GFX1164-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-NEXT: v_mov_b32_e32 v2, s36 +; GFX1164-NEXT: v_mov_b32_e32 v3, s37 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[38:39] +; GFX1164-NEXT: s_cbranch_execnz .LBB6_2 +; GFX1164-NEXT: .LBB6_3: +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: v_mov_b32_e32 v40, v0 +; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1132-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1132-NEXT: s_mov_b32 s38, 0 +; GFX1132-NEXT: s_mov_b32 s32, 32 +; GFX1132-NEXT: s_mov_b32 s0, exec_lo +; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-NEXT: s_cbranch_execz .LBB6_3 +; GFX1132-NEXT: ; %bb.1: +; GFX1132-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1132-NEXT: s_mov_b32 s33, s15 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0 +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-NEXT: .p2align 6 +; GFX1132-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-NEXT: v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-NEXT: s_mov_b32 s12, s33 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX1132-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s36 +; GFX1132-NEXT: v_dual_mov_b32 v3, s37 :: v_dual_mov_b32 v4, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38 +; GFX1132-NEXT: s_cbranch_execnz .LBB6_2 +; GFX1132-NEXT: .LBB6_3: +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s42, -1 +; GFX9-DPP-NEXT: s_mov_b32 s43, 0xe00000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX9-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX9-DPP-NEXT: s_cbranch_execz .LBB6_3 +; GFX9-DPP-NEXT: ; %bb.1: +; GFX9-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX9-DPP-NEXT: s_mov_b32 s33, s2 +; GFX9-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-DPP-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX9-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX9-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX9-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX9-DPP-NEXT: s_mov_b32 s12, s33 +; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX9-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX9-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX9-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX9-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB6_2 +; GFX9-DPP-NEXT: .LBB6_3: +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s42, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s43, 0x31e16000 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1064-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX1064-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1064-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX1064-DPP-NEXT: s_cbranch_execz .LBB6_3 +; GFX1064-DPP-NEXT: ; %bb.1: +; GFX1064-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1064-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1064-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1064-DPP-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX1064-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1064-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1064-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1064-DPP-NEXT: s_clause 0x1 +; GFX1064-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1064-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB6_2 +; GFX1064-DPP-NEXT: .LBB6_3: +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s42, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s43, 0x31c16000 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX1032-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1032-DPP-NEXT: s_mov_b32 s38, 0 +; GFX1032-DPP-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo +; GFX1032-DPP-NEXT: s_cbranch_execz .LBB6_3 +; GFX1032-DPP-NEXT: ; %bb.1: +; GFX1032-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1032-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1032-DPP-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX1032-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1032-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1032-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1032-DPP-NEXT: s_clause 0x1 +; GFX1032-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1032-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-DPP-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s38 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB6_2 +; GFX1032-DPP-NEXT: .LBB6_3: +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1164-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-DPP-NEXT: s_cbranch_execz .LBB6_3 +; GFX1164-DPP-NEXT: ; %bb.1: +; GFX1164-DPP-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1164-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1164-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-DPP-NEXT: .p2align 6 +; GFX1164-DPP-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1164-DPP-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[38:39] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB6_2 +; GFX1164-DPP-NEXT: .LBB6_3: +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s38, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-DPP-NEXT: s_cbranch_execz .LBB6_3 +; GFX1132-DPP-NEXT: ; %bb.1: +; GFX1132-DPP-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1132-DPP-NEXT: s_mov_b32 s33, s15 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0 +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-DPP-NEXT: .p2align 6 +; GFX1132-DPP-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX1132-DPP-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s36 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, s37 :: v_dual_mov_b32 v4, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-DPP-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB6_2 +; GFX1132-DPP-NEXT: .LBB6_3: +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-DPP-NEXT: s_endpgm + %result = atomicrmw fmin ptr addrspace(1) %ptr, double 4.0 syncscope("agent") monotonic, align 4 + ret void +} + +define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_movk_i32 s32, 0x800 +; GFX7LESS-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s50, -1 +; GFX7LESS-NEXT: s_mov_b32 s51, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s48, s48, s9 +; GFX7LESS-NEXT: s_addc_u32 s49, s49, 0 +; GFX7LESS-NEXT: s_mov_b32 s33, s8 +; GFX7LESS-NEXT: s_mov_b32 s40, s7 +; GFX7LESS-NEXT: s_mov_b32 s41, s6 +; GFX7LESS-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX7LESS-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX7LESS-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX7LESS-NEXT: s_load_dwordx2 s[44:45], s[2:3], 0x9 +; GFX7LESS-NEXT: s_mov_b32 s47, 0xf000 +; GFX7LESS-NEXT: s_mov_b32 s46, -1 +; GFX7LESS-NEXT: s_add_u32 s8, s36, 44 +; GFX7LESS-NEXT: s_addc_u32 s9, s37, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v40, v0, v2 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX7LESS-NEXT: s_mov_b32 s12, s41 +; GFX7LESS-NEXT: s_mov_b32 s13, s40 +; GFX7LESS-NEXT: s_mov_b32 s14, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v40 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX7LESS-NEXT: buffer_load_dwordx2 v[2:3], off, s[44:47], 0 +; GFX7LESS-NEXT: s_mov_b64 s[42:43], 0 +; GFX7LESS-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX7LESS-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX7LESS-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX7LESS-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX7LESS-NEXT: s_add_u32 s8, s36, 44 +; GFX7LESS-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX7LESS-NEXT: s_addc_u32 s9, s37, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, 0 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX7LESS-NEXT: s_mov_b32 s12, s41 +; GFX7LESS-NEXT: s_mov_b32 s13, s40 +; GFX7LESS-NEXT: s_mov_b32 s14, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v40 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s44 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s45 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX7LESS-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX7LESS-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX7LESS-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX7LESS-NEXT: s_or_b64 s[42:43], vcc, s[42:43] +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[42:43] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB7_1 +; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s50, -1 +; GFX9-NEXT: s_mov_b32 s51, 0xe00000 +; GFX9-NEXT: s_add_u32 s48, s48, s9 +; GFX9-NEXT: s_addc_u32 s49, s49, 0 +; GFX9-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX9-NEXT: s_mov_b32 s33, s8 +; GFX9-NEXT: s_add_u32 s8, s36, 44 +; GFX9-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX9-NEXT: s_mov_b32 s40, s7 +; GFX9-NEXT: s_mov_b32 s41, s6 +; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX9-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX9-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-NEXT: s_mov_b32 s12, s41 +; GFX9-NEXT: s_mov_b32 s13, s40 +; GFX9-NEXT: s_mov_b32 s14, s33 +; GFX9-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-NEXT: s_movk_i32 s32, 0x800 +; GFX9-NEXT: v_mov_b32_e32 v41, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX9-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX9-NEXT: s_mov_b64 s[44:45], 0 +; GFX9-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX9-NEXT: s_add_u32 s8, s36, 44 +; GFX9-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX9-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX9-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX9-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-NEXT: s_mov_b32 s12, s41 +; GFX9-NEXT: s_mov_b32 s13, s40 +; GFX9-NEXT: s_mov_b32 s14, s33 +; GFX9-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-NEXT: v_mov_b32_e32 v2, s42 +; GFX9-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX9-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX9-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: v_mov_b32_e32 v3, s43 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX9-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX9-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX9-NEXT: s_cbranch_execnz .LBB7_1 +; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s50, -1 +; GFX1064-NEXT: s_mov_b32 s51, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s48, s48, s9 +; GFX1064-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1064-NEXT: s_addc_u32 s49, s49, 0 +; GFX1064-NEXT: s_mov_b32 s33, s8 +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1064-NEXT: s_mov_b32 s40, s7 +; GFX1064-NEXT: s_mov_b32 s41, s6 +; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1064-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1064-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX1064-NEXT: s_mov_b32 s12, s41 +; GFX1064-NEXT: s_mov_b32 s13, s40 +; GFX1064-NEXT: s_mov_b32 s14, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-NEXT: v_mov_b32_e32 v41, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX1064-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1064-NEXT: s_mov_b64 s[44:45], 0 +; GFX1064-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX1064-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-NEXT: v_mov_b32_e32 v2, s42 +; GFX1064-NEXT: v_mov_b32_e32 v3, s43 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-NEXT: s_mov_b32 s12, s41 +; GFX1064-NEXT: s_mov_b32 s13, s40 +; GFX1064-NEXT: s_mov_b32 s14, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX1064-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX1064-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX1064-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-NEXT: s_clause 0x1 +; GFX1064-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX1064-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX1064-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX1064-NEXT: s_cbranch_execnz .LBB7_1 +; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s50, -1 +; GFX1032-NEXT: s_mov_b32 s51, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s48, s48, s9 +; GFX1032-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1032-NEXT: s_addc_u32 s49, s49, 0 +; GFX1032-NEXT: s_mov_b32 s33, s8 +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1032-NEXT: s_mov_b32 s40, s7 +; GFX1032-NEXT: s_mov_b32 s41, s6 +; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1032-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1032-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX1032-NEXT: s_mov_b32 s12, s41 +; GFX1032-NEXT: s_mov_b32 s13, s40 +; GFX1032-NEXT: s_mov_b32 s14, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-NEXT: v_mov_b32_e32 v41, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX1032-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1032-NEXT: s_mov_b32 s44, 0 +; GFX1032-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX1032-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-NEXT: v_mov_b32_e32 v2, s42 +; GFX1032-NEXT: v_mov_b32_e32 v3, s43 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-NEXT: s_mov_b32 s12, s41 +; GFX1032-NEXT: s_mov_b32 s13, s40 +; GFX1032-NEXT: s_mov_b32 s14, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX1032-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX1032-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX1032-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-NEXT: s_clause 0x1 +; GFX1032-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX1032-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX1032-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s44 +; GFX1032-NEXT: s_cbranch_execnz .LBB7_1 +; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1164-NEXT: s_mov_b32 s33, s8 +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1164-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-NEXT: s_mov_b32 s12, s6 +; GFX1164-NEXT: s_mov_b32 s13, s7 +; GFX1164-NEXT: s_mov_b32 s14, s33 +; GFX1164-NEXT: s_mov_b32 s32, 32 +; GFX1164-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-NEXT: s_mov_b32 s40, s7 +; GFX1164-NEXT: s_mov_b32 s41, s6 +; GFX1164-NEXT: v_mov_b32_e32 v41, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: global_load_b64 v[2:3], v41, s[42:43] +; GFX1164-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1164-NEXT: s_mov_b64 s[44:45], 0 +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-NEXT: .p2align 6 +; GFX1164-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-NEXT: s_mov_b32 s12, s41 +; GFX1164-NEXT: s_mov_b32 s13, s40 +; GFX1164-NEXT: s_mov_b32 s14, s33 +; GFX1164-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b64 off, v[2:3], off +; GFX1164-NEXT: scratch_store_b64 off, v[0:1], off offset:8 +; GFX1164-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-NEXT: v_mov_b32_e32 v2, s42 +; GFX1164-NEXT: v_mov_b32_e32 v3, s43 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: scratch_load_b64 v[2:3], off, off +; GFX1164-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[44:45] +; GFX1164-NEXT: s_cbranch_execnz .LBB7_1 +; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1132-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1132-NEXT: v_mov_b32_e32 v31, v0 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1132-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1132-NEXT: s_mov_b32 s40, s14 +; GFX1132-NEXT: s_mov_b32 s41, s13 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-NEXT: s_mov_b32 s12, s13 +; GFX1132-NEXT: s_mov_b32 s13, s14 +; GFX1132-NEXT: s_mov_b32 s14, s15 +; GFX1132-NEXT: s_mov_b32 s32, 32 +; GFX1132-NEXT: s_mov_b32 s33, s15 +; GFX1132-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v41, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: global_load_b64 v[2:3], v41, s[42:43] +; GFX1132-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1132-NEXT: s_mov_b32 s44, 0 +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-NEXT: .p2align 6 +; GFX1132-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-NEXT: v_mov_b32_e32 v31, v40 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-NEXT: s_mov_b32 s12, s41 +; GFX1132-NEXT: s_mov_b32 s13, s40 +; GFX1132-NEXT: s_mov_b32 s14, s33 +; GFX1132-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b64 off, v[2:3], off +; GFX1132-NEXT: scratch_store_b64 off, v[0:1], off offset:8 +; GFX1132-NEXT: v_dual_mov_b32 v0, 8 :: v_dual_mov_b32 v1, 0 +; GFX1132-NEXT: v_dual_mov_b32 v2, s42 :: v_dual_mov_b32 v3, s43 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: scratch_load_b64 v[2:3], off, off +; GFX1132-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s44 +; GFX1132-NEXT: s_cbranch_execnz .LBB7_1 +; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s50, -1 +; GFX9-DPP-NEXT: s_mov_b32 s51, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX9-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX9-DPP-NEXT: s_mov_b32 s33, s8 +; GFX9-DPP-NEXT: s_add_u32 s8, s36, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_mov_b32 s40, s7 +; GFX9-DPP-NEXT: s_mov_b32 s41, s6 +; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-DPP-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX9-DPP-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-DPP-NEXT: s_mov_b32 s12, s41 +; GFX9-DPP-NEXT: s_mov_b32 s13, s40 +; GFX9-DPP-NEXT: s_mov_b32 s14, s33 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX9-DPP-NEXT: v_mov_b32_e32 v41, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-DPP-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX9-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX9-DPP-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX9-DPP-NEXT: s_add_u32 s8, s36, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX9-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-DPP-NEXT: s_mov_b32 s12, s41 +; GFX9-DPP-NEXT: s_mov_b32 s13, s40 +; GFX9-DPP-NEXT: s_mov_b32 s14, s33 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX9-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX9-DPP-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX9-DPP-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX9-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB7_1 +; GFX9-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s50, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s51, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX1064-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1064-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1064-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-DPP-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX1064-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v41, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-DPP-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX1064-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1064-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX1064-DPP-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX1064-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX1064-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX1064-DPP-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-DPP-NEXT: s_clause 0x1 +; GFX1064-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX1064-DPP-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX1064-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB7_1 +; GFX1064-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s50, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s51, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX1032-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1032-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1032-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-DPP-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX1032-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-DPP-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v41, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-DPP-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX1032-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1032-DPP-NEXT: s_mov_b32 s44, 0 +; GFX1032-DPP-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX1032-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX1032-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX1032-DPP-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-DPP-NEXT: s_clause 0x1 +; GFX1032-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX1032-DPP-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX1032-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-DPP-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s44 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB7_1 +; GFX1032-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1164-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v41, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: global_load_b64 v[2:3], v41, s[42:43] +; GFX1164-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1164-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-DPP-NEXT: .p2align 6 +; GFX1164-DPP-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1164-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1164-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[2:3], off +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[0:1], off offset:8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: scratch_load_b64 v[2:3], off, off +; GFX1164-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[44:45] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB7_1 +; GFX1164-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1132-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1132-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1132-DPP-NEXT: s_mov_b32 s40, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s41, s13 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-DPP-NEXT: s_mov_b32 s12, s13 +; GFX1132-DPP-NEXT: s_mov_b32 s13, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s15 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1132-DPP-NEXT: s_mov_b32 s33, s15 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v41, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: global_load_b64 v[2:3], v41, s[42:43] +; GFX1132-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s44, 0 +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-DPP-NEXT: .p2align 6 +; GFX1132-DPP-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1132-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[2:3], off +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[0:1], off offset:8 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 8 :: v_dual_mov_b32 v1, 0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s42 :: v_dual_mov_b32 v3, s43 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: scratch_load_b64 v[2:3], off, off +; GFX1132-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-DPP-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s44 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB7_1 +; GFX1132-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-DPP-NEXT: s_endpgm + %divValue = call double @div.double.value() + %result = atomicrmw fmin ptr addrspace(1) %ptr, double %divValue syncscope("agent") monotonic, align 4 + ret void +} + +define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_one_as_scope_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fmin_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0 +; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX7LESS-NEXT: s_cbranch_execz .LBB8_3 +; GFX7LESS-NEXT: ; %bb.1: +; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0 +; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s6 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s7 +; GFX7LESS-NEXT: s_mov_b32 s2, -1 +; GFX7LESS-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX7LESS-NEXT: v_min_f64 v[0:1], v[0:1], 4.0 +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, v1 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, v0 +; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3] +; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5] +; GFX7LESS-NEXT: v_mov_b32_e32 v2, v4 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, v5 +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB8_2 +; GFX7LESS-NEXT: .LBB8_3: +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fmin_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX9-NEXT: s_cbranch_execz .LBB8_3 +; GFX9-NEXT: ; %bb.1: +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-NEXT: s_mov_b64 s[2:3], 0 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v2, s4 +; GFX9-NEXT: v_mov_b32_e32 v3, s5 +; GFX9-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX9-NEXT: v_min_f64 v[0:1], v[0:1], 4.0 +; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX9-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX9-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX9-NEXT: s_cbranch_execnz .LBB8_2 +; GFX9-NEXT: .LBB8_3: +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fmin_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX1064-NEXT: s_cbranch_execz .LBB8_3 +; GFX1064-NEXT: ; %bb.1: +; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: v_mov_b32_e32 v2, s2 +; GFX1064-NEXT: v_mov_b32_e32 v3, s3 +; GFX1064-NEXT: s_mov_b64 s[2:3], 0 +; GFX1064-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1064-NEXT: v_min_f64 v[0:1], v[0:1], 4.0 +; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1064-NEXT: v_mov_b32_e32 v3, v1 +; GFX1064-NEXT: v_mov_b32_e32 v2, v0 +; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX1064-NEXT: s_cbranch_execnz .LBB8_2 +; GFX1064-NEXT: .LBB8_3: +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fmin_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1032-NEXT: s_mov_b32 s2, 0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-NEXT: s_and_saveexec_b32 s3, vcc_lo +; GFX1032-NEXT: s_cbranch_execz .LBB8_3 +; GFX1032-NEXT: ; %bb.1: +; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: v_mov_b32_e32 v2, s4 +; GFX1032-NEXT: v_mov_b32_e32 v3, s5 +; GFX1032-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1032-NEXT: v_min_f64 v[0:1], v[0:1], 4.0 +; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1032-NEXT: v_mov_b32_e32 v3, v1 +; GFX1032-NEXT: v_mov_b32_e32 v2, v0 +; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 +; GFX1032-NEXT: s_cbranch_execnz .LBB8_2 +; GFX1032-NEXT: .LBB8_3: +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fmin_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1164-NEXT: s_mov_b64 s[2:3], exec +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-NEXT: s_cbranch_execz .LBB8_3 +; GFX1164-NEXT: ; %bb.1: +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_load_b64 s[2:3], s[0:1], 0x0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: v_mov_b32_e32 v2, s2 +; GFX1164-NEXT: v_mov_b32_e32 v3, s3 +; GFX1164-NEXT: s_mov_b64 s[2:3], 0 +; GFX1164-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1164-NEXT: v_min_f64 v[0:1], v[0:1], 4.0 +; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1164-NEXT: v_mov_b32_e32 v3, v1 +; GFX1164-NEXT: v_mov_b32_e32 v2, v0 +; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3] +; GFX1164-NEXT: s_cbranch_execnz .LBB8_2 +; GFX1164-NEXT: .LBB8_3: +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fmin_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1132-NEXT: s_mov_b32 s2, 0 +; GFX1132-NEXT: s_mov_b32 s3, exec_lo +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-NEXT: s_cbranch_execz .LBB8_3 +; GFX1132-NEXT: ; %bb.1: +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1132-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_load_b64 s[4:5], s[0:1], 0x0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5 +; GFX1132-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1132-NEXT: v_min_f64 v[0:1], v[0:1], 4.0 +; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2 +; GFX1132-NEXT: s_cbranch_execnz .LBB8_2 +; GFX1132-NEXT: .LBB8_3: +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX9-DPP-NEXT: s_cbranch_execz .LBB8_3 +; GFX9-DPP-NEXT: ; %bb.1: +; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s4 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s5 +; GFX9-DPP-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX9-DPP-NEXT: v_min_f64 v[0:1], v[0:1], 4.0 +; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB8_2 +; GFX9-DPP-NEXT: .LBB8_3: +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX1064-DPP-NEXT: s_cbranch_execz .LBB8_3 +; GFX1064-DPP-NEXT: ; %bb.1: +; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s2 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s3 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX1064-DPP-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1064-DPP-NEXT: v_min_f64 v[0:1], v[0:1], 4.0 +; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB8_2 +; GFX1064-DPP-NEXT: .LBB8_3: +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s2, 0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-DPP-NEXT: s_and_saveexec_b32 s3, vcc_lo +; GFX1032-DPP-NEXT: s_cbranch_execz .LBB8_3 +; GFX1032-DPP-NEXT: ; %bb.1: +; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s4 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s5 +; GFX1032-DPP-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1032-DPP-NEXT: v_min_f64 v[0:1], v[0:1], 4.0 +; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB8_2 +; GFX1032-DPP-NEXT: .LBB8_3: +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-DPP-NEXT: s_cbranch_execz .LBB8_3 +; GFX1164-DPP-NEXT: ; %bb.1: +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s2 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s3 +; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX1164-DPP-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1164-DPP-NEXT: v_min_f64 v[0:1], v[0:1], 4.0 +; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB8_2 +; GFX1164-DPP-NEXT: .LBB8_3: +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s2, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s3, exec_lo +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-DPP-NEXT: s_cbranch_execz .LBB8_3 +; GFX1132-DPP-NEXT: ; %bb.1: +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5 +; GFX1132-DPP-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1132-DPP-NEXT: v_min_f64 v[0:1], v[0:1], 4.0 +; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB8_2 +; GFX1132-DPP-NEXT: .LBB8_3: +; GFX1132-DPP-NEXT: s_endpgm + %result = atomicrmw fmin ptr addrspace(1) %ptr, double 4.0 syncscope("one-as") monotonic + ret void +} + +define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_as_scope_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fmin_double_uni_address_div_value_one_as_scope_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_mov_b32 s32, 0 +; GFX7LESS-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s42, -1 +; GFX7LESS-NEXT: s_mov_b32 s43, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s40, s40, s9 +; GFX7LESS-NEXT: s_addc_u32 s41, s41, 0 +; GFX7LESS-NEXT: s_mov_b32 s14, s8 +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX7LESS-NEXT: s_load_dwordx2 s[36:37], s[2:3], 0x9 +; GFX7LESS-NEXT: s_mov_b32 s39, 0xf000 +; GFX7LESS-NEXT: s_mov_b32 s38, -1 +; GFX7LESS-NEXT: s_add_u32 s8, s2, 44 +; GFX7LESS-NEXT: s_addc_u32 s9, s3, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[2:3] +; GFX7LESS-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v31, v0, v2 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX7LESS-NEXT: s_mov_b32 s12, s6 +; GFX7LESS-NEXT: s_mov_b32 s13, s7 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX7LESS-NEXT: buffer_load_dwordx2 v[2:3], off, s[36:39], 0 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], 0 +; GFX7LESS-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX7LESS-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX7LESS-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3 +; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0 +; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[36:39], 0 glc +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3] +; GFX7LESS-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7 +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB9_1 +; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fmin_double_uni_address_div_value_one_as_scope_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s38, -1 +; GFX9-NEXT: s_mov_b32 s39, 0xe00000 +; GFX9-NEXT: s_add_u32 s36, s36, s9 +; GFX9-NEXT: s_addc_u32 s37, s37, 0 +; GFX9-NEXT: s_mov_b32 s14, s8 +; GFX9-NEXT: s_add_u32 s8, s2, 44 +; GFX9-NEXT: s_addc_u32 s9, s3, 0 +; GFX9-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX9-NEXT: s_getpc_b64 s[2:3] +; GFX9-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX9-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX9-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX9-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX9-NEXT: s_mov_b32 s12, s6 +; GFX9-NEXT: s_mov_b32 s13, s7 +; GFX9-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX9-NEXT: s_mov_b32 s32, 0 +; GFX9-NEXT: v_mov_b32_e32 v40, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX9-NEXT: global_load_dwordx2 v[2:3], v40, s[34:35] +; GFX9-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX9-NEXT: s_mov_b64 s[0:1], 0 +; GFX9-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX9-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5] +; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v40, v[0:3], s[34:35] glc +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX9-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX9-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX9-NEXT: s_cbranch_execnz .LBB9_1 +; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fmin_double_uni_address_div_value_one_as_scope_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s38, -1 +; GFX1064-NEXT: s_mov_b32 s39, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s36, s36, s9 +; GFX1064-NEXT: s_addc_u32 s37, s37, 0 +; GFX1064-NEXT: s_mov_b32 s14, s8 +; GFX1064-NEXT: s_add_u32 s8, s2, 44 +; GFX1064-NEXT: s_addc_u32 s9, s3, 0 +; GFX1064-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1064-NEXT: s_getpc_b64 s[4:5] +; GFX1064-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1064-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1064-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1064-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1064-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1064-NEXT: s_mov_b32 s12, s6 +; GFX1064-NEXT: s_mov_b32 s13, s7 +; GFX1064-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1064-NEXT: s_mov_b32 s32, 0 +; GFX1064-NEXT: v_mov_b32_e32 v40, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1064-NEXT: global_load_dwordx2 v[2:3], v40, s[34:35] +; GFX1064-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX1064-NEXT: s_mov_b64 s[0:1], 0 +; GFX1064-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1064-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5] +; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v40, v[0:3], s[34:35] glc +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1064-NEXT: v_mov_b32_e32 v3, v1 +; GFX1064-NEXT: v_mov_b32_e32 v2, v0 +; GFX1064-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX1064-NEXT: s_cbranch_execnz .LBB9_1 +; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fmin_double_uni_address_div_value_one_as_scope_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s38, -1 +; GFX1032-NEXT: s_mov_b32 s39, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s36, s36, s9 +; GFX1032-NEXT: s_addc_u32 s37, s37, 0 +; GFX1032-NEXT: s_mov_b32 s14, s8 +; GFX1032-NEXT: s_add_u32 s8, s2, 44 +; GFX1032-NEXT: s_addc_u32 s9, s3, 0 +; GFX1032-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1032-NEXT: s_getpc_b64 s[4:5] +; GFX1032-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1032-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1032-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1032-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1032-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1032-NEXT: s_mov_b32 s12, s6 +; GFX1032-NEXT: s_mov_b32 s13, s7 +; GFX1032-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1032-NEXT: s_mov_b32 s32, 0 +; GFX1032-NEXT: v_mov_b32_e32 v40, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1032-NEXT: global_load_dwordx2 v[2:3], v40, s[34:35] +; GFX1032-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX1032-NEXT: s_mov_b32 s0, 0 +; GFX1032-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1032-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5] +; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v40, v[0:3], s[34:35] glc +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1032-NEXT: v_mov_b32_e32 v3, v1 +; GFX1032-NEXT: v_mov_b32_e32 v2, v0 +; GFX1032-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 +; GFX1032-NEXT: s_cbranch_execnz .LBB9_1 +; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fmin_double_uni_address_div_value_one_as_scope_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b32 s14, s8 +; GFX1164-NEXT: s_add_u32 s8, s2, 44 +; GFX1164-NEXT: s_addc_u32 s9, s3, 0 +; GFX1164-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1164-NEXT: s_getpc_b64 s[4:5] +; GFX1164-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1164-NEXT: s_load_b64 s[16:17], s[4:5], 0x0 +; GFX1164-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1164-NEXT: s_mov_b32 s12, s6 +; GFX1164-NEXT: s_mov_b32 s13, s7 +; GFX1164-NEXT: s_mov_b32 s32, 0 +; GFX1164-NEXT: v_mov_b32_e32 v40, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1164-NEXT: global_load_b64 v[2:3], v40, s[34:35] +; GFX1164-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX1164-NEXT: s_mov_b64 s[0:1], 0 +; GFX1164-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5] +; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v40, v[0:3], s[34:35] glc +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1164-NEXT: v_mov_b32_e32 v3, v1 +; GFX1164-NEXT: v_mov_b32_e32 v2, v0 +; GFX1164-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[0:1] +; GFX1164-NEXT: s_cbranch_execnz .LBB9_1 +; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fmin_double_uni_address_div_value_one_as_scope_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_add_u32 s8, s2, 44 +; GFX1132-NEXT: s_addc_u32 s9, s3, 0 +; GFX1132-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1132-NEXT: s_getpc_b64 s[4:5] +; GFX1132-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1132-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 +; GFX1132-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1132-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v31, v0 +; GFX1132-NEXT: s_mov_b32 s12, s13 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1132-NEXT: s_mov_b32 s13, s14 +; GFX1132-NEXT: s_mov_b32 s14, s15 +; GFX1132-NEXT: s_mov_b32 s32, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1132-NEXT: global_load_b64 v[2:3], v40, s[34:35] +; GFX1132-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX1132-NEXT: s_mov_b32 s0, 0 +; GFX1132-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5] +; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v40, v[0:3], s[34:35] glc +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX1132-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX1132-NEXT: s_cbranch_execnz .LBB9_1 +; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_one_as_scope_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s38, -1 +; GFX9-DPP-NEXT: s_mov_b32 s39, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX9-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX9-DPP-NEXT: s_mov_b32 s14, s8 +; GFX9-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX9-DPP-NEXT: s_getpc_b64 s[2:3] +; GFX9-DPP-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX9-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX9-DPP-NEXT: s_mov_b32 s12, s6 +; GFX9-DPP-NEXT: s_mov_b32 s13, s7 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX9-DPP-NEXT: s_mov_b32 s32, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX9-DPP-NEXT: global_load_dwordx2 v[2:3], v40, s[34:35] +; GFX9-DPP-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX9-DPP-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX9-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5] +; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v40, v[0:3], s[34:35] glc +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB9_1 +; GFX9-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_one_as_scope_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s38, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s39, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX1064-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1064-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1064-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1064-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1064-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1064-DPP-NEXT: global_load_dwordx2 v[2:3], v40, s[34:35] +; GFX1064-DPP-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX1064-DPP-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1064-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5] +; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v40, v[0:3], s[34:35] glc +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB9_1 +; GFX1064-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_one_as_scope_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s38, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s39, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX1032-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1032-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1032-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1032-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1032-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1032-DPP-NEXT: global_load_dwordx2 v[2:3], v40, s[34:35] +; GFX1032-DPP-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX1032-DPP-NEXT: s_mov_b32 s0, 0 +; GFX1032-DPP-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1032-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5] +; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v40, v[0:3], s[34:35] glc +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1032-DPP-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB9_1 +; GFX1032-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_one_as_scope_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1164-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1164-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1164-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: s_load_b64 s[16:17], s[4:5], 0x0 +; GFX1164-DPP-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1164-DPP-NEXT: global_load_b64 v[2:3], v40, s[34:35] +; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX1164-DPP-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5] +; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v40, v[0:3], s[34:35] glc +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[0:1] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB9_1 +; GFX1164-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_one_as_scope_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1132-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1132-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 +; GFX1132-DPP-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v31, v0 +; GFX1132-DPP-NEXT: s_mov_b32 s12, s13 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s13, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s15 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1132-DPP-NEXT: global_load_b64 v[2:3], v40, s[34:35] +; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s0, 0 +; GFX1132-DPP-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5] +; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v40, v[0:3], s[34:35] glc +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX1132-DPP-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB9_1 +; GFX1132-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-DPP-NEXT: s_endpgm + %divValue = call double @div.double.value() + %result = atomicrmw fmin ptr addrspace(1) %ptr, double %divValue syncscope("one-as") monotonic + ret void +} + +define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defalut_scope_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fmin_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_movk_i32 s32, 0x800 +; GFX7LESS-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s42, -1 +; GFX7LESS-NEXT: s_mov_b32 s43, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s40, s40, s3 +; GFX7LESS-NEXT: s_addc_u32 s41, s41, 0 +; GFX7LESS-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX7LESS-NEXT: v_mov_b32_e32 v40, v0 +; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0 +; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX7LESS-NEXT: s_cbranch_execz .LBB10_3 +; GFX7LESS-NEXT: ; %bb.1: +; GFX7LESS-NEXT: s_mov_b32 s33, s2 +; GFX7LESS-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x9 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX7LESS-NEXT: s_mov_b64 s[38:39], 0 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, s0 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, s1 +; GFX7LESS-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_max_f64 v[2:3], v[0:1], v[0:1] +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[40:43], 0 offset:4 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[40:43], 0 +; GFX7LESS-NEXT: s_add_u32 s8, s34, 44 +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_min_f64 v[0:1], v[2:3], 4.0 +; GFX7LESS-NEXT: s_addc_u32 s9, s35, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[40:43], 0 offset:12 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[40:43], 0 offset:8 +; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, 0 +; GFX7LESS-NEXT: s_mov_b32 s12, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v40 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s36 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s37 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX7LESS-NEXT: v_and_b32_e32 v2, 1, v0 +; GFX7LESS-NEXT: buffer_load_dword v0, off, s[40:43], 0 +; GFX7LESS-NEXT: buffer_load_dword v1, off, s[40:43], 0 offset:4 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2 +; GFX7LESS-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB10_2 +; GFX7LESS-NEXT: .LBB10_3: +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fmin_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s42, -1 +; GFX9-NEXT: s_mov_b32 s43, 0xe00000 +; GFX9-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX9-NEXT: s_add_u32 s40, s40, s3 +; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-NEXT: s_addc_u32 s41, s41, 0 +; GFX9-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-NEXT: s_movk_i32 s32, 0x800 +; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX9-NEXT: s_cbranch_execz .LBB10_3 +; GFX9-NEXT: ; %bb.1: +; GFX9-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX9-NEXT: s_mov_b32 s33, s2 +; GFX9-NEXT: s_mov_b64 s[38:39], 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX9-NEXT: s_add_u32 s8, s34, 44 +; GFX9-NEXT: s_addc_u32 s9, s35, 0 +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX9-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX9-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX9-NEXT: s_mov_b32 s12, s33 +; GFX9-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX9-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX9-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: v_mov_b32_e32 v2, s36 +; GFX9-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX9-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX9-NEXT: v_mov_b32_e32 v3, s37 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX9-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX9-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX9-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX9-NEXT: s_cbranch_execnz .LBB10_2 +; GFX9-NEXT: .LBB10_3: +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fmin_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1064-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s42, -1 +; GFX1064-NEXT: s_mov_b32 s43, 0x31e16000 +; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1064-NEXT: s_add_u32 s40, s40, s3 +; GFX1064-NEXT: s_addc_u32 s41, s41, 0 +; GFX1064-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1064-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX1064-NEXT: s_cbranch_execz .LBB10_3 +; GFX1064-NEXT: ; %bb.1: +; GFX1064-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1064-NEXT: s_mov_b32 s33, s2 +; GFX1064-NEXT: s_mov_b64 s[38:39], 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: v_mov_b32_e32 v2, s1 +; GFX1064-NEXT: v_mov_b32_e32 v1, s0 +; GFX1064-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1064-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1064-NEXT: s_mov_b32 s12, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1064-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX1064-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1064-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1064-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1064-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-NEXT: v_mov_b32_e32 v2, s36 +; GFX1064-NEXT: v_mov_b32_e32 v3, s37 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1064-NEXT: s_clause 0x1 +; GFX1064-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1064-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX1064-NEXT: s_cbranch_execnz .LBB10_2 +; GFX1064-NEXT: .LBB10_3: +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fmin_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1032-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s42, -1 +; GFX1032-NEXT: s_mov_b32 s43, 0x31c16000 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-NEXT: s_add_u32 s40, s40, s3 +; GFX1032-NEXT: s_addc_u32 s41, s41, 0 +; GFX1032-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1032-NEXT: s_mov_b32 s38, 0 +; GFX1032-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo +; GFX1032-NEXT: s_cbranch_execz .LBB10_3 +; GFX1032-NEXT: ; %bb.1: +; GFX1032-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1032-NEXT: s_mov_b32 s33, s2 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: v_mov_b32_e32 v2, s1 +; GFX1032-NEXT: v_mov_b32_e32 v1, s0 +; GFX1032-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1032-NEXT: s_mov_b32 s12, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1032-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX1032-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1032-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1032-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1032-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-NEXT: v_mov_b32_e32 v2, s36 +; GFX1032-NEXT: v_mov_b32_e32 v3, s37 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1032-NEXT: s_clause 0x1 +; GFX1032-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1032-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s38 +; GFX1032-NEXT: s_cbranch_execnz .LBB10_2 +; GFX1032-NEXT: .LBB10_3: +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fmin_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1164-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1164-NEXT: s_mov_b32 s32, 32 +; GFX1164-NEXT: s_mov_b64 s[0:1], exec +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-NEXT: s_cbranch_execz .LBB10_3 +; GFX1164-NEXT: ; %bb.1: +; GFX1164-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1164-NEXT: s_mov_b32 s33, s2 +; GFX1164-NEXT: s_mov_b64 s[38:39], 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: v_mov_b32_e32 v2, s1 +; GFX1164-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-NEXT: .p2align 6 +; GFX1164-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-NEXT: s_mov_b32 s12, s33 +; GFX1164-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-NEXT: v_mov_b32_e32 v2, s36 +; GFX1164-NEXT: v_mov_b32_e32 v3, s37 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[38:39] +; GFX1164-NEXT: s_cbranch_execnz .LBB10_2 +; GFX1164-NEXT: .LBB10_3: +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fmin_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: v_mov_b32_e32 v40, v0 +; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1132-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1132-NEXT: s_mov_b32 s38, 0 +; GFX1132-NEXT: s_mov_b32 s32, 32 +; GFX1132-NEXT: s_mov_b32 s0, exec_lo +; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-NEXT: s_cbranch_execz .LBB10_3 +; GFX1132-NEXT: ; %bb.1: +; GFX1132-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1132-NEXT: s_mov_b32 s33, s15 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0 +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-NEXT: .p2align 6 +; GFX1132-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-NEXT: v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-NEXT: s_mov_b32 s12, s33 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX1132-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s36 +; GFX1132-NEXT: v_dual_mov_b32 v3, s37 :: v_dual_mov_b32 v4, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38 +; GFX1132-NEXT: s_cbranch_execnz .LBB10_2 +; GFX1132-NEXT: .LBB10_3: +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s42, -1 +; GFX9-DPP-NEXT: s_mov_b32 s43, 0xe00000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX9-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX9-DPP-NEXT: s_cbranch_execz .LBB10_3 +; GFX9-DPP-NEXT: ; %bb.1: +; GFX9-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX9-DPP-NEXT: s_mov_b32 s33, s2 +; GFX9-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-DPP-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX9-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX9-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX9-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX9-DPP-NEXT: s_mov_b32 s12, s33 +; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX9-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX9-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX9-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX9-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB10_2 +; GFX9-DPP-NEXT: .LBB10_3: +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s42, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s43, 0x31e16000 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1064-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX1064-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1064-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX1064-DPP-NEXT: s_cbranch_execz .LBB10_3 +; GFX1064-DPP-NEXT: ; %bb.1: +; GFX1064-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1064-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1064-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1064-DPP-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX1064-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1064-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1064-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1064-DPP-NEXT: s_clause 0x1 +; GFX1064-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1064-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB10_2 +; GFX1064-DPP-NEXT: .LBB10_3: +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s42, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s43, 0x31c16000 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX1032-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1032-DPP-NEXT: s_mov_b32 s38, 0 +; GFX1032-DPP-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo +; GFX1032-DPP-NEXT: s_cbranch_execz .LBB10_3 +; GFX1032-DPP-NEXT: ; %bb.1: +; GFX1032-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1032-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1032-DPP-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX1032-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1032-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1032-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1032-DPP-NEXT: s_clause 0x1 +; GFX1032-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1032-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-DPP-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s38 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB10_2 +; GFX1032-DPP-NEXT: .LBB10_3: +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1164-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-DPP-NEXT: s_cbranch_execz .LBB10_3 +; GFX1164-DPP-NEXT: ; %bb.1: +; GFX1164-DPP-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1164-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1164-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-DPP-NEXT: .p2align 6 +; GFX1164-DPP-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1164-DPP-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[38:39] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB10_2 +; GFX1164-DPP-NEXT: .LBB10_3: +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s38, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-DPP-NEXT: s_cbranch_execz .LBB10_3 +; GFX1132-DPP-NEXT: ; %bb.1: +; GFX1132-DPP-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1132-DPP-NEXT: s_mov_b32 s33, s15 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0 +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-DPP-NEXT: .p2align 6 +; GFX1132-DPP-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX1132-DPP-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s36 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, s37 :: v_dual_mov_b32 v4, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-DPP-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB10_2 +; GFX1132-DPP-NEXT: .LBB10_3: +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-DPP-NEXT: s_endpgm + %result = atomicrmw fmin ptr addrspace(1) %ptr, double 4.0 monotonic, align 4 + ret void +} + +define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defalut_scope_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fmin_double_uni_address_div_value_defalut_scope_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_movk_i32 s32, 0x800 +; GFX7LESS-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s50, -1 +; GFX7LESS-NEXT: s_mov_b32 s51, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s48, s48, s9 +; GFX7LESS-NEXT: s_addc_u32 s49, s49, 0 +; GFX7LESS-NEXT: s_mov_b32 s33, s8 +; GFX7LESS-NEXT: s_mov_b32 s40, s7 +; GFX7LESS-NEXT: s_mov_b32 s41, s6 +; GFX7LESS-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX7LESS-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX7LESS-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX7LESS-NEXT: s_load_dwordx2 s[44:45], s[2:3], 0x9 +; GFX7LESS-NEXT: s_mov_b32 s47, 0xf000 +; GFX7LESS-NEXT: s_mov_b32 s46, -1 +; GFX7LESS-NEXT: s_add_u32 s8, s36, 44 +; GFX7LESS-NEXT: s_addc_u32 s9, s37, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v40, v0, v2 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX7LESS-NEXT: s_mov_b32 s12, s41 +; GFX7LESS-NEXT: s_mov_b32 s13, s40 +; GFX7LESS-NEXT: s_mov_b32 s14, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v40 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX7LESS-NEXT: buffer_load_dwordx2 v[2:3], off, s[44:47], 0 +; GFX7LESS-NEXT: s_mov_b64 s[42:43], 0 +; GFX7LESS-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX7LESS-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX7LESS-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX7LESS-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX7LESS-NEXT: s_add_u32 s8, s36, 44 +; GFX7LESS-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX7LESS-NEXT: s_addc_u32 s9, s37, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, 0 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX7LESS-NEXT: s_mov_b32 s12, s41 +; GFX7LESS-NEXT: s_mov_b32 s13, s40 +; GFX7LESS-NEXT: s_mov_b32 s14, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v40 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s44 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s45 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX7LESS-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX7LESS-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX7LESS-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX7LESS-NEXT: s_or_b64 s[42:43], vcc, s[42:43] +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[42:43] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB11_1 +; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fmin_double_uni_address_div_value_defalut_scope_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s50, -1 +; GFX9-NEXT: s_mov_b32 s51, 0xe00000 +; GFX9-NEXT: s_add_u32 s48, s48, s9 +; GFX9-NEXT: s_addc_u32 s49, s49, 0 +; GFX9-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX9-NEXT: s_mov_b32 s33, s8 +; GFX9-NEXT: s_add_u32 s8, s36, 44 +; GFX9-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX9-NEXT: s_mov_b32 s40, s7 +; GFX9-NEXT: s_mov_b32 s41, s6 +; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX9-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX9-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-NEXT: s_mov_b32 s12, s41 +; GFX9-NEXT: s_mov_b32 s13, s40 +; GFX9-NEXT: s_mov_b32 s14, s33 +; GFX9-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-NEXT: s_movk_i32 s32, 0x800 +; GFX9-NEXT: v_mov_b32_e32 v41, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX9-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX9-NEXT: s_mov_b64 s[44:45], 0 +; GFX9-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX9-NEXT: s_add_u32 s8, s36, 44 +; GFX9-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX9-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX9-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX9-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-NEXT: s_mov_b32 s12, s41 +; GFX9-NEXT: s_mov_b32 s13, s40 +; GFX9-NEXT: s_mov_b32 s14, s33 +; GFX9-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-NEXT: v_mov_b32_e32 v2, s42 +; GFX9-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX9-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX9-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: v_mov_b32_e32 v3, s43 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX9-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX9-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX9-NEXT: s_cbranch_execnz .LBB11_1 +; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fmin_double_uni_address_div_value_defalut_scope_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s50, -1 +; GFX1064-NEXT: s_mov_b32 s51, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s48, s48, s9 +; GFX1064-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1064-NEXT: s_addc_u32 s49, s49, 0 +; GFX1064-NEXT: s_mov_b32 s33, s8 +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1064-NEXT: s_mov_b32 s40, s7 +; GFX1064-NEXT: s_mov_b32 s41, s6 +; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1064-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1064-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX1064-NEXT: s_mov_b32 s12, s41 +; GFX1064-NEXT: s_mov_b32 s13, s40 +; GFX1064-NEXT: s_mov_b32 s14, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-NEXT: v_mov_b32_e32 v41, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX1064-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1064-NEXT: s_mov_b64 s[44:45], 0 +; GFX1064-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX1064-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-NEXT: v_mov_b32_e32 v2, s42 +; GFX1064-NEXT: v_mov_b32_e32 v3, s43 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-NEXT: s_mov_b32 s12, s41 +; GFX1064-NEXT: s_mov_b32 s13, s40 +; GFX1064-NEXT: s_mov_b32 s14, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX1064-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX1064-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX1064-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-NEXT: s_clause 0x1 +; GFX1064-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX1064-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX1064-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX1064-NEXT: s_cbranch_execnz .LBB11_1 +; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fmin_double_uni_address_div_value_defalut_scope_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s50, -1 +; GFX1032-NEXT: s_mov_b32 s51, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s48, s48, s9 +; GFX1032-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1032-NEXT: s_addc_u32 s49, s49, 0 +; GFX1032-NEXT: s_mov_b32 s33, s8 +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1032-NEXT: s_mov_b32 s40, s7 +; GFX1032-NEXT: s_mov_b32 s41, s6 +; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1032-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1032-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX1032-NEXT: s_mov_b32 s12, s41 +; GFX1032-NEXT: s_mov_b32 s13, s40 +; GFX1032-NEXT: s_mov_b32 s14, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-NEXT: v_mov_b32_e32 v41, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX1032-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1032-NEXT: s_mov_b32 s44, 0 +; GFX1032-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX1032-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-NEXT: v_mov_b32_e32 v2, s42 +; GFX1032-NEXT: v_mov_b32_e32 v3, s43 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-NEXT: s_mov_b32 s12, s41 +; GFX1032-NEXT: s_mov_b32 s13, s40 +; GFX1032-NEXT: s_mov_b32 s14, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX1032-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX1032-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX1032-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-NEXT: s_clause 0x1 +; GFX1032-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX1032-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX1032-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s44 +; GFX1032-NEXT: s_cbranch_execnz .LBB11_1 +; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fmin_double_uni_address_div_value_defalut_scope_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1164-NEXT: s_mov_b32 s33, s8 +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1164-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-NEXT: s_mov_b32 s12, s6 +; GFX1164-NEXT: s_mov_b32 s13, s7 +; GFX1164-NEXT: s_mov_b32 s14, s33 +; GFX1164-NEXT: s_mov_b32 s32, 32 +; GFX1164-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-NEXT: s_mov_b32 s40, s7 +; GFX1164-NEXT: s_mov_b32 s41, s6 +; GFX1164-NEXT: v_mov_b32_e32 v41, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: global_load_b64 v[2:3], v41, s[42:43] +; GFX1164-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1164-NEXT: s_mov_b64 s[44:45], 0 +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-NEXT: .p2align 6 +; GFX1164-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-NEXT: s_mov_b32 s12, s41 +; GFX1164-NEXT: s_mov_b32 s13, s40 +; GFX1164-NEXT: s_mov_b32 s14, s33 +; GFX1164-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b64 off, v[2:3], off +; GFX1164-NEXT: scratch_store_b64 off, v[0:1], off offset:8 +; GFX1164-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-NEXT: v_mov_b32_e32 v2, s42 +; GFX1164-NEXT: v_mov_b32_e32 v3, s43 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: scratch_load_b64 v[2:3], off, off +; GFX1164-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[44:45] +; GFX1164-NEXT: s_cbranch_execnz .LBB11_1 +; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fmin_double_uni_address_div_value_defalut_scope_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1132-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1132-NEXT: v_mov_b32_e32 v31, v0 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1132-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1132-NEXT: s_mov_b32 s40, s14 +; GFX1132-NEXT: s_mov_b32 s41, s13 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-NEXT: s_mov_b32 s12, s13 +; GFX1132-NEXT: s_mov_b32 s13, s14 +; GFX1132-NEXT: s_mov_b32 s14, s15 +; GFX1132-NEXT: s_mov_b32 s32, 32 +; GFX1132-NEXT: s_mov_b32 s33, s15 +; GFX1132-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v41, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: global_load_b64 v[2:3], v41, s[42:43] +; GFX1132-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1132-NEXT: s_mov_b32 s44, 0 +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-NEXT: .p2align 6 +; GFX1132-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-NEXT: v_mov_b32_e32 v31, v40 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-NEXT: s_mov_b32 s12, s41 +; GFX1132-NEXT: s_mov_b32 s13, s40 +; GFX1132-NEXT: s_mov_b32 s14, s33 +; GFX1132-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b64 off, v[2:3], off +; GFX1132-NEXT: scratch_store_b64 off, v[0:1], off offset:8 +; GFX1132-NEXT: v_dual_mov_b32 v0, 8 :: v_dual_mov_b32 v1, 0 +; GFX1132-NEXT: v_dual_mov_b32 v2, s42 :: v_dual_mov_b32 v3, s43 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: scratch_load_b64 v[2:3], off, off +; GFX1132-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s44 +; GFX1132-NEXT: s_cbranch_execnz .LBB11_1 +; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_defalut_scope_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s50, -1 +; GFX9-DPP-NEXT: s_mov_b32 s51, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX9-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX9-DPP-NEXT: s_mov_b32 s33, s8 +; GFX9-DPP-NEXT: s_add_u32 s8, s36, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_mov_b32 s40, s7 +; GFX9-DPP-NEXT: s_mov_b32 s41, s6 +; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-DPP-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX9-DPP-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-DPP-NEXT: s_mov_b32 s12, s41 +; GFX9-DPP-NEXT: s_mov_b32 s13, s40 +; GFX9-DPP-NEXT: s_mov_b32 s14, s33 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX9-DPP-NEXT: v_mov_b32_e32 v41, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-DPP-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX9-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX9-DPP-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX9-DPP-NEXT: s_add_u32 s8, s36, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX9-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-DPP-NEXT: s_mov_b32 s12, s41 +; GFX9-DPP-NEXT: s_mov_b32 s13, s40 +; GFX9-DPP-NEXT: s_mov_b32 s14, s33 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX9-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX9-DPP-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX9-DPP-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX9-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB11_1 +; GFX9-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_defalut_scope_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s50, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s51, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX1064-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1064-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1064-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-DPP-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX1064-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v41, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-DPP-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX1064-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1064-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX1064-DPP-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX1064-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX1064-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX1064-DPP-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-DPP-NEXT: s_clause 0x1 +; GFX1064-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX1064-DPP-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX1064-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB11_1 +; GFX1064-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_defalut_scope_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s50, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s51, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX1032-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1032-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1032-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-DPP-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX1032-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-DPP-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v41, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-DPP-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX1032-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1032-DPP-NEXT: s_mov_b32 s44, 0 +; GFX1032-DPP-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX1032-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX1032-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX1032-DPP-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-DPP-NEXT: s_clause 0x1 +; GFX1032-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX1032-DPP-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX1032-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-DPP-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s44 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB11_1 +; GFX1032-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_defalut_scope_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1164-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v41, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: global_load_b64 v[2:3], v41, s[42:43] +; GFX1164-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1164-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-DPP-NEXT: .p2align 6 +; GFX1164-DPP-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1164-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1164-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[2:3], off +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[0:1], off offset:8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: scratch_load_b64 v[2:3], off, off +; GFX1164-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[44:45] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB11_1 +; GFX1164-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_defalut_scope_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1132-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1132-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1132-DPP-NEXT: s_mov_b32 s40, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s41, s13 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-DPP-NEXT: s_mov_b32 s12, s13 +; GFX1132-DPP-NEXT: s_mov_b32 s13, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s15 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1132-DPP-NEXT: s_mov_b32 s33, s15 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v41, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: global_load_b64 v[2:3], v41, s[42:43] +; GFX1132-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s44, 0 +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-DPP-NEXT: .p2align 6 +; GFX1132-DPP-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1132-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[2:3], off +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[0:1], off offset:8 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 8 :: v_dual_mov_b32 v1, 0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s42 :: v_dual_mov_b32 v3, s43 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: scratch_load_b64 v[2:3], off, off +; GFX1132-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-DPP-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s44 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB11_1 +; GFX1132-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-DPP-NEXT: s_endpgm + %divValue = call double @div.double.value() + %result = atomicrmw fmin ptr addrspace(1) %ptr, double %divValue monotonic, align 4 + ret void +} + attributes #0 = { "denormal-fp-math-f32"="preserve-sign,preserve-sign" "amdgpu-unsafe-fp-atomics"="true" } !llvm.module.flags = !{!0} diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll index bc9125e326c4..7a7ddbe618b0 100644 --- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll +++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll @@ -13,6 +13,7 @@ ; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,-wavefrontsize64 -amdgpu-atomic-optimizer-strategy=DPP -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX1132-DPP %s declare float @div.float.value() +declare double @div.double.value() define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 { ; GFX7LESS-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_unsafe: @@ -5616,6 +5617,5581 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_defalut_scop ret void } +define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_movk_i32 s32, 0x800 +; GFX7LESS-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s42, -1 +; GFX7LESS-NEXT: s_mov_b32 s43, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s40, s40, s3 +; GFX7LESS-NEXT: s_addc_u32 s41, s41, 0 +; GFX7LESS-NEXT: s_mov_b32 s33, s2 +; GFX7LESS-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX7LESS-NEXT: v_mov_b32_e32 v40, v0 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec +; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s0, 0 +; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s1, v0 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX7LESS-NEXT: s_cbranch_execz .LBB9_3 +; GFX7LESS-NEXT: ; %bb.1: +; GFX7LESS-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x9 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_load_dwordx2 s[2:3], s[36:37], 0x0 +; GFX7LESS-NEXT: s_bcnt1_i32_b64 s0, s[0:1] +; GFX7LESS-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX7LESS-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX7LESS-NEXT: s_mov_b64 s[38:39], 0 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, s2 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, s3 +; GFX7LESS-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_add_f64 v[2:3], v[0:1], -v[41:42] +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[40:43], 0 offset:4 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[40:43], 0 +; GFX7LESS-NEXT: s_add_u32 s8, s34, 44 +; GFX7LESS-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:12 +; GFX7LESS-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:8 +; GFX7LESS-NEXT: s_addc_u32 s9, s35, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX7LESS-NEXT: s_waitcnt expcnt(2) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, 0 +; GFX7LESS-NEXT: s_mov_b32 s12, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v40 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s36 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s37 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX7LESS-NEXT: v_and_b32_e32 v2, 1, v0 +; GFX7LESS-NEXT: buffer_load_dword v0, off, s[40:43], 0 +; GFX7LESS-NEXT: buffer_load_dword v1, off, s[40:43], 0 offset:4 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2 +; GFX7LESS-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB9_2 +; GFX7LESS-NEXT: .LBB9_3: +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s42, -1 +; GFX9-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX9-NEXT: s_mov_b64 s[0:1], exec +; GFX9-NEXT: s_mov_b32 s43, 0xe00000 +; GFX9-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0 +; GFX9-NEXT: s_add_u32 s40, s40, s3 +; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0 +; GFX9-NEXT: s_addc_u32 s41, s41, 0 +; GFX9-NEXT: s_mov_b32 s33, s2 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-NEXT: s_movk_i32 s32, 0x800 +; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX9-NEXT: s_cbranch_execz .LBB9_3 +; GFX9-NEXT: ; %bb.1: +; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1] +; GFX9-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX9-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX9-NEXT: s_mov_b64 s[38:39], 0 +; GFX9-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX9-NEXT: s_add_u32 s8, s34, 44 +; GFX9-NEXT: s_addc_u32 s9, s35, 0 +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX9-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX9-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX9-NEXT: s_mov_b32 s12, s33 +; GFX9-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX9-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX9-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX9-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: v_mov_b32_e32 v2, s36 +; GFX9-NEXT: v_mov_b32_e32 v3, s37 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX9-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX9-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX9-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX9-NEXT: s_cbranch_execnz .LBB9_2 +; GFX9-NEXT: .LBB9_3: +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s42, -1 +; GFX1064-NEXT: s_mov_b32 s43, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s40, s40, s3 +; GFX1064-NEXT: s_mov_b32 s33, s2 +; GFX1064-NEXT: s_mov_b64 s[2:3], exec +; GFX1064-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1064-NEXT: s_addc_u32 s41, s41, 0 +; GFX1064-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1064-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX1064-NEXT: s_cbranch_execz .LBB9_3 +; GFX1064-NEXT: ; %bb.1: +; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[2:3] +; GFX1064-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1064-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX1064-NEXT: s_mov_b64 s[38:39], 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1064-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: v_mov_b32_e32 v2, s1 +; GFX1064-NEXT: v_mov_b32_e32 v1, s0 +; GFX1064-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1064-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1064-NEXT: s_mov_b32 s12, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1064-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1064-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1064-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1064-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-NEXT: v_mov_b32_e32 v2, s36 +; GFX1064-NEXT: v_mov_b32_e32 v3, s37 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1064-NEXT: s_clause 0x1 +; GFX1064-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1064-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX1064-NEXT: s_cbranch_execnz .LBB9_2 +; GFX1064-NEXT: .LBB9_3: +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s33, s2 +; GFX1032-NEXT: s_mov_b32 s2, exec_lo +; GFX1032-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1032-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s42, -1 +; GFX1032-NEXT: s_mov_b32 s43, 0x31c16000 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-NEXT: s_add_u32 s40, s40, s3 +; GFX1032-NEXT: s_addc_u32 s41, s41, 0 +; GFX1032-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1032-NEXT: s_mov_b32 s38, 0 +; GFX1032-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo +; GFX1032-NEXT: s_cbranch_execz .LBB9_3 +; GFX1032-NEXT: ; %bb.1: +; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s2 +; GFX1032-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1032-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1032-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: v_mov_b32_e32 v2, s1 +; GFX1032-NEXT: v_mov_b32_e32 v1, s0 +; GFX1032-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1032-NEXT: s_mov_b32 s12, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1032-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1032-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1032-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1032-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-NEXT: v_mov_b32_e32 v2, s36 +; GFX1032-NEXT: v_mov_b32_e32 v3, s37 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1032-NEXT: s_clause 0x1 +; GFX1032-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1032-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s38 +; GFX1032-NEXT: s_cbranch_execnz .LBB9_2 +; GFX1032-NEXT: .LBB9_3: +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b32 s33, s2 +; GFX1164-NEXT: s_mov_b64 s[2:3], exec +; GFX1164-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1164-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1164-NEXT: s_mov_b32 s32, 32 +; GFX1164-NEXT: s_mov_b64 s[0:1], exec +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-NEXT: s_cbranch_execz .LBB9_3 +; GFX1164-NEXT: ; %bb.1: +; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[2:3] +; GFX1164-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1164-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX1164-NEXT: s_mov_b64 s[38:39], 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: v_mov_b32_e32 v2, s1 +; GFX1164-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-NEXT: .p2align 6 +; GFX1164-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-NEXT: s_mov_b32 s12, s33 +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-NEXT: v_mov_b32_e32 v2, s36 +; GFX1164-NEXT: v_mov_b32_e32 v3, s37 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[38:39] +; GFX1164-NEXT: s_cbranch_execnz .LBB9_2 +; GFX1164-NEXT: .LBB9_3: +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_mov_b32 s2, exec_lo +; GFX1132-NEXT: v_mov_b32_e32 v40, v0 +; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1132-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1132-NEXT: s_mov_b32 s38, 0 +; GFX1132-NEXT: s_mov_b32 s32, 32 +; GFX1132-NEXT: s_mov_b32 s0, exec_lo +; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-NEXT: s_cbranch_execz .LBB9_3 +; GFX1132-NEXT: ; %bb.1: +; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s2 +; GFX1132-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1132-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX1132-NEXT: s_mov_b32 s33, s15 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0 +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-NEXT: .p2align 6 +; GFX1132-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-NEXT: v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-NEXT: s_mov_b32 s12, s33 +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s36 +; GFX1132-NEXT: v_dual_mov_b32 v3, s37 :: v_dual_mov_b32 v4, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38 +; GFX1132-NEXT: s_cbranch_execnz .LBB9_2 +; GFX1132-NEXT: .LBB9_3: +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s42, -1 +; GFX9-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], exec +; GFX9-DPP-NEXT: s_mov_b32 s43, 0xe00000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0 +; GFX9-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0 +; GFX9-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX9-DPP-NEXT: s_mov_b32 s33, s2 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX9-DPP-NEXT: s_cbranch_execz .LBB9_3 +; GFX9-DPP-NEXT: ; %bb.1: +; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1] +; GFX9-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX9-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX9-DPP-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-DPP-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX9-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX9-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX9-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX9-DPP-NEXT: s_mov_b32 s12, s33 +; GFX9-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX9-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX9-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX9-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX9-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB9_2 +; GFX9-DPP-NEXT: .LBB9_3: +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s42, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s43, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX1064-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1064-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1064-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX1064-DPP-NEXT: s_cbranch_execz .LBB9_3 +; GFX1064-DPP-NEXT: ; %bb.1: +; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[2:3] +; GFX1064-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1064-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX1064-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1064-DPP-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1064-DPP-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1064-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1064-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1064-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1064-DPP-NEXT: s_clause 0x1 +; GFX1064-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1064-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB9_2 +; GFX1064-DPP-NEXT: .LBB9_3: +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1032-DPP-NEXT: s_mov_b32 s2, exec_lo +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s42, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s43, 0x31c16000 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX1032-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1032-DPP-NEXT: s_mov_b32 s38, 0 +; GFX1032-DPP-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo +; GFX1032-DPP-NEXT: s_cbranch_execz .LBB9_3 +; GFX1032-DPP-NEXT: ; %bb.1: +; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s2 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1032-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1032-DPP-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1032-DPP-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1032-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1032-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1032-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1032-DPP-NEXT: s_clause 0x1 +; GFX1032-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1032-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-DPP-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s38 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB9_2 +; GFX1032-DPP-NEXT: .LBB9_3: +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1164-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-DPP-NEXT: s_cbranch_execz .LBB9_3 +; GFX1164-DPP-NEXT: ; %bb.1: +; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[2:3] +; GFX1164-DPP-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1164-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX1164-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-DPP-NEXT: .p2align 6 +; GFX1164-DPP-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[38:39] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB9_2 +; GFX1164-DPP-NEXT: .LBB9_3: +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_mov_b32 s2, exec_lo +; GFX1132-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s38, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-DPP-NEXT: s_cbranch_execz .LBB9_3 +; GFX1132-DPP-NEXT: ; %bb.1: +; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s2 +; GFX1132-DPP-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1132-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX1132-DPP-NEXT: s_mov_b32 s33, s15 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0 +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-DPP-NEXT: .p2align 6 +; GFX1132-DPP-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s36 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, s37 :: v_dual_mov_b32 v4, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-DPP-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB9_2 +; GFX1132-DPP-NEXT: .LBB9_3: +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-DPP-NEXT: s_endpgm + %result = atomicrmw fsub ptr addrspace(1) %ptr, double 4.0 syncscope("agent") monotonic, align 4 + ret void +} + +define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_movk_i32 s32, 0x800 +; GFX7LESS-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s50, -1 +; GFX7LESS-NEXT: s_mov_b32 s51, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s48, s48, s9 +; GFX7LESS-NEXT: s_addc_u32 s49, s49, 0 +; GFX7LESS-NEXT: s_mov_b32 s33, s8 +; GFX7LESS-NEXT: s_mov_b32 s40, s7 +; GFX7LESS-NEXT: s_mov_b32 s41, s6 +; GFX7LESS-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX7LESS-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX7LESS-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX7LESS-NEXT: s_load_dwordx2 s[44:45], s[2:3], 0x9 +; GFX7LESS-NEXT: s_mov_b32 s47, 0xf000 +; GFX7LESS-NEXT: s_mov_b32 s46, -1 +; GFX7LESS-NEXT: s_add_u32 s8, s36, 44 +; GFX7LESS-NEXT: s_addc_u32 s9, s37, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v42, v0, v2 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX7LESS-NEXT: s_mov_b32 s12, s41 +; GFX7LESS-NEXT: s_mov_b32 s13, s40 +; GFX7LESS-NEXT: s_mov_b32 s14, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v42 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX7LESS-NEXT: v_mov_b32_e32 v40, v0 +; GFX7LESS-NEXT: v_mov_b32_e32 v41, v1 +; GFX7LESS-NEXT: buffer_load_dwordx2 v[0:1], off, s[44:47], 0 +; GFX7LESS-NEXT: s_mov_b64 s[42:43], 0 +; GFX7LESS-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_add_f64 v[2:3], v[0:1], -v[40:41] +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:4 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[48:51], 0 +; GFX7LESS-NEXT: s_add_u32 s8, s36, 44 +; GFX7LESS-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:12 +; GFX7LESS-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:8 +; GFX7LESS-NEXT: s_addc_u32 s9, s37, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: s_waitcnt expcnt(2) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, 0 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX7LESS-NEXT: s_mov_b32 s12, s41 +; GFX7LESS-NEXT: s_mov_b32 s13, s40 +; GFX7LESS-NEXT: s_mov_b32 s14, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v42 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s44 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s45 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX7LESS-NEXT: v_and_b32_e32 v2, 1, v0 +; GFX7LESS-NEXT: buffer_load_dword v0, off, s[48:51], 0 +; GFX7LESS-NEXT: buffer_load_dword v1, off, s[48:51], 0 offset:4 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2 +; GFX7LESS-NEXT: s_or_b64 s[42:43], vcc, s[42:43] +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[42:43] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB10_1 +; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s50, -1 +; GFX9-NEXT: s_mov_b32 s51, 0xe00000 +; GFX9-NEXT: s_add_u32 s48, s48, s9 +; GFX9-NEXT: s_addc_u32 s49, s49, 0 +; GFX9-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX9-NEXT: s_mov_b32 s33, s8 +; GFX9-NEXT: s_add_u32 s8, s36, 44 +; GFX9-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX9-NEXT: s_mov_b32 s40, s7 +; GFX9-NEXT: s_mov_b32 s41, s6 +; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX9-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX9-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-NEXT: s_mov_b32 s12, s41 +; GFX9-NEXT: s_mov_b32 s13, s40 +; GFX9-NEXT: s_mov_b32 s14, s33 +; GFX9-NEXT: v_mov_b32_e32 v31, v42 +; GFX9-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-NEXT: s_movk_i32 s32, 0x800 +; GFX9-NEXT: v_mov_b32_e32 v43, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-NEXT: v_mov_b32_e32 v41, v1 +; GFX9-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX9-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-NEXT: s_mov_b64 s[44:45], 0 +; GFX9-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX9-NEXT: s_add_u32 s8, s36, 44 +; GFX9-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX9-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX9-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX9-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX9-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-NEXT: s_mov_b32 s12, s41 +; GFX9-NEXT: s_mov_b32 s13, s40 +; GFX9-NEXT: s_mov_b32 s14, s33 +; GFX9-NEXT: v_mov_b32_e32 v31, v42 +; GFX9-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: v_mov_b32_e32 v2, s42 +; GFX9-NEXT: v_mov_b32_e32 v3, s43 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX9-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX9-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX9-NEXT: s_cbranch_execnz .LBB10_1 +; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s50, -1 +; GFX1064-NEXT: s_mov_b32 s51, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s48, s48, s9 +; GFX1064-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1064-NEXT: s_addc_u32 s49, s49, 0 +; GFX1064-NEXT: s_mov_b32 s33, s8 +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1064-NEXT: s_mov_b32 s40, s7 +; GFX1064-NEXT: s_mov_b32 s41, s6 +; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1064-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1064-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX1064-NEXT: s_mov_b32 s12, s41 +; GFX1064-NEXT: s_mov_b32 s13, s40 +; GFX1064-NEXT: s_mov_b32 s14, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-NEXT: v_mov_b32_e32 v31, v42 +; GFX1064-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-NEXT: v_mov_b32_e32 v43, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-NEXT: v_mov_b32_e32 v41, v1 +; GFX1064-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX1064-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-NEXT: s_mov_b64 s[44:45], 0 +; GFX1064-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX1064-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-NEXT: v_mov_b32_e32 v31, v42 +; GFX1064-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-NEXT: v_mov_b32_e32 v2, s42 +; GFX1064-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-NEXT: s_mov_b32 s12, s41 +; GFX1064-NEXT: s_mov_b32 s13, s40 +; GFX1064-NEXT: s_mov_b32 s14, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX1064-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX1064-NEXT: v_mov_b32_e32 v3, s43 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-NEXT: s_clause 0x1 +; GFX1064-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX1064-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX1064-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX1064-NEXT: s_cbranch_execnz .LBB10_1 +; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s50, -1 +; GFX1032-NEXT: s_mov_b32 s51, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s48, s48, s9 +; GFX1032-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1032-NEXT: s_addc_u32 s49, s49, 0 +; GFX1032-NEXT: s_mov_b32 s33, s8 +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1032-NEXT: s_mov_b32 s40, s7 +; GFX1032-NEXT: s_mov_b32 s41, s6 +; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1032-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1032-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX1032-NEXT: s_mov_b32 s12, s41 +; GFX1032-NEXT: s_mov_b32 s13, s40 +; GFX1032-NEXT: s_mov_b32 s14, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-NEXT: v_mov_b32_e32 v31, v42 +; GFX1032-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-NEXT: v_mov_b32_e32 v43, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-NEXT: v_mov_b32_e32 v41, v1 +; GFX1032-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX1032-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-NEXT: s_mov_b32 s44, 0 +; GFX1032-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX1032-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-NEXT: v_mov_b32_e32 v31, v42 +; GFX1032-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-NEXT: v_mov_b32_e32 v2, s42 +; GFX1032-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-NEXT: s_mov_b32 s12, s41 +; GFX1032-NEXT: s_mov_b32 s13, s40 +; GFX1032-NEXT: s_mov_b32 s14, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX1032-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX1032-NEXT: v_mov_b32_e32 v3, s43 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-NEXT: s_clause 0x1 +; GFX1032-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX1032-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX1032-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s44 +; GFX1032-NEXT: s_cbranch_execnz .LBB10_1 +; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1164-NEXT: s_mov_b32 s33, s8 +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1164-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-NEXT: s_mov_b32 s12, s6 +; GFX1164-NEXT: s_mov_b32 s13, s7 +; GFX1164-NEXT: s_mov_b32 s14, s33 +; GFX1164-NEXT: s_mov_b32 s32, 32 +; GFX1164-NEXT: v_mov_b32_e32 v42, v0 +; GFX1164-NEXT: s_mov_b32 s40, s7 +; GFX1164-NEXT: s_mov_b32 s41, s6 +; GFX1164-NEXT: v_mov_b32_e32 v43, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: v_mov_b32_e32 v41, v1 +; GFX1164-NEXT: global_load_b64 v[1:2], v43, s[42:43] +; GFX1164-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-NEXT: s_mov_b64 s[44:45], 0 +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-NEXT: .p2align 6 +; GFX1164-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-NEXT: v_mov_b32_e32 v31, v42 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-NEXT: s_mov_b32 s12, s41 +; GFX1164-NEXT: s_mov_b32 s13, s40 +; GFX1164-NEXT: s_mov_b32 s14, s33 +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-NEXT: v_mov_b32_e32 v2, s42 +; GFX1164-NEXT: v_mov_b32_e32 v3, s43 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[44:45] +; GFX1164-NEXT: s_cbranch_execnz .LBB10_1 +; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1132-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1132-NEXT: v_mov_b32_e32 v31, v0 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1132-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1132-NEXT: s_mov_b32 s40, s14 +; GFX1132-NEXT: s_mov_b32 s41, s13 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-NEXT: s_mov_b32 s12, s13 +; GFX1132-NEXT: s_mov_b32 s13, s14 +; GFX1132-NEXT: s_mov_b32 s14, s15 +; GFX1132-NEXT: s_mov_b32 s32, 32 +; GFX1132-NEXT: s_mov_b32 s33, s15 +; GFX1132-NEXT: v_dual_mov_b32 v42, v0 :: v_dual_mov_b32 v43, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v41, v1 +; GFX1132-NEXT: global_load_b64 v[1:2], v43, s[42:43] +; GFX1132-NEXT: s_mov_b32 s44, 0 +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-NEXT: .p2align 6 +; GFX1132-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-NEXT: v_dual_mov_b32 v31, v42 :: v_dual_mov_b32 v0, 8 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-NEXT: s_mov_b32 s12, s41 +; GFX1132-NEXT: s_mov_b32 s13, s40 +; GFX1132-NEXT: s_mov_b32 s14, s33 +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s42 +; GFX1132-NEXT: v_dual_mov_b32 v3, s43 :: v_dual_mov_b32 v4, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s44 +; GFX1132-NEXT: s_cbranch_execnz .LBB10_1 +; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s50, -1 +; GFX9-DPP-NEXT: s_mov_b32 s51, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX9-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX9-DPP-NEXT: s_mov_b32 s33, s8 +; GFX9-DPP-NEXT: s_add_u32 s8, s36, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_mov_b32 s40, s7 +; GFX9-DPP-NEXT: s_mov_b32 s41, s6 +; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-DPP-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX9-DPP-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-DPP-NEXT: s_mov_b32 s12, s41 +; GFX9-DPP-NEXT: s_mov_b32 s13, s40 +; GFX9-DPP-NEXT: s_mov_b32 s14, s33 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX9-DPP-NEXT: v_mov_b32_e32 v43, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-DPP-NEXT: v_mov_b32_e32 v41, v1 +; GFX9-DPP-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX9-DPP-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX9-DPP-NEXT: s_add_u32 s8, s36, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX9-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-DPP-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX9-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-DPP-NEXT: s_mov_b32 s12, s41 +; GFX9-DPP-NEXT: s_mov_b32 s13, s40 +; GFX9-DPP-NEXT: s_mov_b32 s14, s33 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-DPP-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX9-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX9-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB10_1 +; GFX9-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s50, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s51, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX1064-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1064-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1064-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-DPP-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX1064-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1064-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v43, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v41, v1 +; GFX1064-DPP-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX1064-DPP-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX1064-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-DPP-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX1064-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-DPP-NEXT: s_clause 0x1 +; GFX1064-DPP-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX1064-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX1064-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB10_1 +; GFX1064-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s50, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s51, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX1032-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1032-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1032-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-DPP-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX1032-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1032-DPP-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v43, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v41, v1 +; GFX1032-DPP-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-DPP-NEXT: s_mov_b32 s44, 0 +; GFX1032-DPP-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX1032-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-DPP-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX1032-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-DPP-NEXT: s_clause 0x1 +; GFX1032-DPP-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX1032-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX1032-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-DPP-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s44 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB10_1 +; GFX1032-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1164-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v42, v0 +; GFX1164-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v43, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v41, v1 +; GFX1164-DPP-NEXT: global_load_b64 v[1:2], v43, s[42:43] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-DPP-NEXT: .p2align 6 +; GFX1164-DPP-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1164-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[44:45] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB10_1 +; GFX1164-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1132-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1132-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1132-DPP-NEXT: s_mov_b32 s40, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s41, s13 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-DPP-NEXT: s_mov_b32 s12, s13 +; GFX1132-DPP-NEXT: s_mov_b32 s13, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s15 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1132-DPP-NEXT: s_mov_b32 s33, s15 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v42, v0 :: v_dual_mov_b32 v43, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v41, v1 +; GFX1132-DPP-NEXT: global_load_b64 v[1:2], v43, s[42:43] +; GFX1132-DPP-NEXT: s_mov_b32 s44, 0 +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-DPP-NEXT: .p2align 6 +; GFX1132-DPP-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v31, v42 :: v_dual_mov_b32 v0, 8 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1132-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s42 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, s43 :: v_dual_mov_b32 v4, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-DPP-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s44 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB10_1 +; GFX1132-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-DPP-NEXT: s_endpgm + %divValue = call double @div.float.value() + %result = atomicrmw fsub ptr addrspace(1) %ptr, double %divValue syncscope("agent") monotonic, align 4 + ret void +} + +define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp(ptr addrspace(1) %ptr) #1 { +; GFX7LESS-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s14, -1 +; GFX7LESS-NEXT: s_mov_b32 s15, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s12, s12, s3 +; GFX7LESS-NEXT: s_addc_u32 s13, s13, 0 +; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec +; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0 +; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX7LESS-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX7LESS-NEXT: s_cbranch_execz .LBB11_3 +; GFX7LESS-NEXT: ; %bb.1: +; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[2:3] +; GFX7LESS-NEXT: s_mov_b32 s7, 0x43300000 +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0 +; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000 +; GFX7LESS-NEXT: v_add_f64 v[0:1], s[6:7], v[0:1] +; GFX7LESS-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s8 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s9 +; GFX7LESS-NEXT: s_mov_b32 s2, -1 +; GFX7LESS-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3 +; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0 +; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3] +; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5] +; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7 +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB11_2 +; GFX7LESS-NEXT: .LBB11_3: +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s10, -1 +; GFX9-NEXT: s_mov_b32 s11, 0xe00000 +; GFX9-NEXT: s_add_u32 s8, s8, s3 +; GFX9-NEXT: s_mov_b64 s[2:3], exec +; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX9-NEXT: s_addc_u32 s9, s9, 0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX9-NEXT: s_cbranch_execz .LBB11_3 +; GFX9-NEXT: ; %bb.1: +; GFX9-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX9-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX9-NEXT: s_mov_b32 s3, 0x43300000 +; GFX9-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1] +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-NEXT: s_mov_b64 s[2:3], 0 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v2, s4 +; GFX9-NEXT: v_mov_b32_e32 v3, s5 +; GFX9-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX9-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX9-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX9-NEXT: s_cbranch_execnz .LBB11_2 +; GFX9-NEXT: .LBB11_3: +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s10, -1 +; GFX1064-NEXT: s_mov_b32 s11, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s8, s8, s3 +; GFX1064-NEXT: s_mov_b64 s[2:3], exec +; GFX1064-NEXT: s_addc_u32 s9, s9, 0 +; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX1064-NEXT: s_cbranch_execz .LBB11_3 +; GFX1064-NEXT: ; %bb.1: +; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX1064-NEXT: s_mov_b32 s3, 0x43300000 +; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1064-NEXT: v_add_f64 v[0:1], 0xc3300000, s[2:3] +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 +; GFX1064-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: v_mov_b32_e32 v2, s2 +; GFX1064-NEXT: v_mov_b32_e32 v3, s3 +; GFX1064-NEXT: s_mov_b64 s[2:3], 0 +; GFX1064-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1064-NEXT: v_mov_b32_e32 v3, v1 +; GFX1064-NEXT: v_mov_b32_e32 v2, v0 +; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX1064-NEXT: s_cbranch_execnz .LBB11_2 +; GFX1064-NEXT: .LBB11_3: +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s10, -1 +; GFX1032-NEXT: s_mov_b32 s11, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s8, s8, s3 +; GFX1032-NEXT: s_mov_b32 s3, exec_lo +; GFX1032-NEXT: s_addc_u32 s9, s9, 0 +; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0 +; GFX1032-NEXT: s_mov_b32 s2, 0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo +; GFX1032-NEXT: s_cbranch_execz .LBB11_3 +; GFX1032-NEXT: ; %bb.1: +; GFX1032-NEXT: s_bcnt1_i32_b32 s4, s3 +; GFX1032-NEXT: s_mov_b32 s5, 0x43300000 +; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[4:5] +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: v_mov_b32_e32 v2, s4 +; GFX1032-NEXT: v_mov_b32_e32 v3, s5 +; GFX1032-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1032-NEXT: v_mov_b32_e32 v3, v1 +; GFX1032-NEXT: v_mov_b32_e32 v2, v0 +; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 +; GFX1032-NEXT: s_cbranch_execnz .LBB11_2 +; GFX1032-NEXT: .LBB11_3: +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_bcnt1_i32_b64 s2, exec +; GFX1164-NEXT: v_mov_b32_e32 v0, 0x43300000 +; GFX1164-NEXT: v_mov_b32_e32 v1, s2 +; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1164-NEXT: s_mov_b64 s[2:3], exec +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b32 off, v0, off offset:4 +; GFX1164-NEXT: scratch_store_b32 off, v1, off +; GFX1164-NEXT: scratch_load_b64 v[0:1], off, off +; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1164-NEXT: s_cbranch_execz .LBB11_3 +; GFX1164-NEXT: ; %bb.1: +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_load_b64 s[2:3], s[0:1], 0x0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1164-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: v_mov_b32_e32 v2, s2 +; GFX1164-NEXT: v_mov_b32_e32 v3, s3 +; GFX1164-NEXT: s_mov_b64 s[2:3], 0 +; GFX1164-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1164-NEXT: v_mov_b32_e32 v3, v1 +; GFX1164-NEXT: v_mov_b32_e32 v2, v0 +; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3] +; GFX1164-NEXT: s_cbranch_execnz .LBB11_2 +; GFX1164-NEXT: .LBB11_3: +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_bcnt1_i32_b32 s2, exec_lo +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s2 +; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1132-NEXT: s_mov_b32 s2, 0 +; GFX1132-NEXT: s_mov_b32 s3, exec_lo +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4 +; GFX1132-NEXT: scratch_store_b32 off, v1, off +; GFX1132-NEXT: scratch_load_b64 v[0:1], off, off +; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1132-NEXT: s_cbranch_execz .LBB11_3 +; GFX1132-NEXT: ; %bb.1: +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1132-NEXT: v_mov_b32_e32 v6, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_load_b64 s[4:5], s[0:1], 0x0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1132-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5 +; GFX1132-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2 +; GFX1132-NEXT: s_cbranch_execnz .LBB11_2 +; GFX1132-NEXT: .LBB11_3: +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s10, -1 +; GFX9-DPP-NEXT: s_mov_b32 s11, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s8, s8, s3 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX9-DPP-NEXT: s_addc_u32 s9, s9, 0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX9-DPP-NEXT: s_cbranch_execz .LBB11_3 +; GFX9-DPP-NEXT: ; %bb.1: +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX9-DPP-NEXT: s_mov_b32 s3, 0x43300000 +; GFX9-DPP-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1] +; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s4 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s5 +; GFX9-DPP-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB11_2 +; GFX9-DPP-NEXT: .LBB11_3: +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s10, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s11, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s8, s8, s3 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX1064-DPP-NEXT: s_addc_u32 s9, s9, 0 +; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX1064-DPP-NEXT: s_cbranch_execz .LBB11_3 +; GFX1064-DPP-NEXT: ; %bb.1: +; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX1064-DPP-NEXT: s_mov_b32 s3, 0x43300000 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1064-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[2:3] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s2 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s3 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX1064-DPP-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB11_2 +; GFX1064-DPP-NEXT: .LBB11_3: +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s10, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s11, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s8, s8, s3 +; GFX1032-DPP-NEXT: s_mov_b32 s3, exec_lo +; GFX1032-DPP-NEXT: s_addc_u32 s9, s9, 0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s2, 0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-DPP-NEXT: s_and_saveexec_b32 s4, vcc_lo +; GFX1032-DPP-NEXT: s_cbranch_execz .LBB11_3 +; GFX1032-DPP-NEXT: ; %bb.1: +; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s4, s3 +; GFX1032-DPP-NEXT: s_mov_b32 s5, 0x43300000 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[4:5] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s4 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s5 +; GFX1032-DPP-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB11_2 +; GFX1032-DPP-NEXT: .LBB11_3: +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s2, exec +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s2 +; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b32 off, v0, off offset:4 +; GFX1164-DPP-NEXT: scratch_store_b32 off, v1, off +; GFX1164-DPP-NEXT: scratch_load_b64 v[0:1], off, off +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1164-DPP-NEXT: s_cbranch_execz .LBB11_3 +; GFX1164-DPP-NEXT: ; %bb.1: +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1164-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s2 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s3 +; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX1164-DPP-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB11_2 +; GFX1164-DPP-NEXT: .LBB11_3: +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s2, exec_lo +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s2 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s2, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s3, exec_lo +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4 +; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off +; GFX1132-DPP-NEXT: scratch_load_b64 v[0:1], off, off +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1132-DPP-NEXT: s_cbranch_execz .LBB11_3 +; GFX1132-DPP-NEXT: ; %bb.1: +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5 +; GFX1132-DPP-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB11_2 +; GFX1132-DPP-NEXT: .LBB11_3: +; GFX1132-DPP-NEXT: s_endpgm + %result = atomicrmw fsub ptr addrspace(1) %ptr, double 4.0 syncscope("one-as") monotonic + ret void +} +define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_one_as_scope_unsafe_structfp(ptr addrspace(1) %ptr) #1 { +; GFX7LESS-LABEL: global_atomic_fsub_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_mov_b32 s32, 0 +; GFX7LESS-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s42, -1 +; GFX7LESS-NEXT: s_mov_b32 s43, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s40, s40, s9 +; GFX7LESS-NEXT: s_addc_u32 s41, s41, 0 +; GFX7LESS-NEXT: s_mov_b32 s14, s8 +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX7LESS-NEXT: s_load_dwordx2 s[36:37], s[2:3], 0x9 +; GFX7LESS-NEXT: s_mov_b32 s39, 0xf000 +; GFX7LESS-NEXT: s_mov_b32 s38, -1 +; GFX7LESS-NEXT: s_add_u32 s8, s2, 44 +; GFX7LESS-NEXT: s_addc_u32 s9, s3, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[2:3] +; GFX7LESS-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v31, v0, v2 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX7LESS-NEXT: s_mov_b32 s12, s6 +; GFX7LESS-NEXT: s_mov_b32 s13, s7 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX7LESS-NEXT: buffer_load_dwordx2 v[4:5], off, s[36:39], 0 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], 0 +; GFX7LESS-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v9, v5 +; GFX7LESS-NEXT: v_mov_b32_e32 v8, v4 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2 +; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[36:39], 0 glc +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[4:5] +; GFX7LESS-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX7LESS-NEXT: v_mov_b32_e32 v4, v6 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, v7 +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB12_1 +; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fsub_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s38, -1 +; GFX9-NEXT: s_mov_b32 s39, 0xe00000 +; GFX9-NEXT: s_add_u32 s36, s36, s9 +; GFX9-NEXT: s_addc_u32 s37, s37, 0 +; GFX9-NEXT: s_mov_b32 s14, s8 +; GFX9-NEXT: s_add_u32 s8, s2, 44 +; GFX9-NEXT: s_addc_u32 s9, s3, 0 +; GFX9-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX9-NEXT: s_getpc_b64 s[2:3] +; GFX9-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX9-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX9-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX9-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX9-NEXT: s_mov_b32 s12, s6 +; GFX9-NEXT: s_mov_b32 s13, s7 +; GFX9-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX9-NEXT: s_mov_b32 s32, 0 +; GFX9-NEXT: v_mov_b32_e32 v40, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX9-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX9-NEXT: s_mov_b64 s[0:1], 0 +; GFX9-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX9-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX9-NEXT: v_mov_b32_e32 v5, v3 +; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX9-NEXT: v_mov_b32_e32 v4, v2 +; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX9-NEXT: s_cbranch_execnz .LBB12_1 +; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fsub_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s38, -1 +; GFX1064-NEXT: s_mov_b32 s39, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s36, s36, s9 +; GFX1064-NEXT: s_addc_u32 s37, s37, 0 +; GFX1064-NEXT: s_mov_b32 s14, s8 +; GFX1064-NEXT: s_add_u32 s8, s2, 44 +; GFX1064-NEXT: s_addc_u32 s9, s3, 0 +; GFX1064-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1064-NEXT: s_getpc_b64 s[4:5] +; GFX1064-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1064-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1064-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1064-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1064-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1064-NEXT: s_mov_b32 s12, s6 +; GFX1064-NEXT: s_mov_b32 s13, s7 +; GFX1064-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1064-NEXT: s_mov_b32 s32, 0 +; GFX1064-NEXT: v_mov_b32_e32 v40, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1064-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1064-NEXT: s_mov_b64 s[0:1], 0 +; GFX1064-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1064-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1064-NEXT: v_mov_b32_e32 v5, v3 +; GFX1064-NEXT: v_mov_b32_e32 v4, v2 +; GFX1064-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX1064-NEXT: s_cbranch_execnz .LBB12_1 +; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fsub_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s38, -1 +; GFX1032-NEXT: s_mov_b32 s39, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s36, s36, s9 +; GFX1032-NEXT: s_addc_u32 s37, s37, 0 +; GFX1032-NEXT: s_mov_b32 s14, s8 +; GFX1032-NEXT: s_add_u32 s8, s2, 44 +; GFX1032-NEXT: s_addc_u32 s9, s3, 0 +; GFX1032-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1032-NEXT: s_getpc_b64 s[4:5] +; GFX1032-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1032-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1032-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1032-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1032-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1032-NEXT: s_mov_b32 s12, s6 +; GFX1032-NEXT: s_mov_b32 s13, s7 +; GFX1032-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1032-NEXT: s_mov_b32 s32, 0 +; GFX1032-NEXT: v_mov_b32_e32 v40, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1032-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1032-NEXT: s_mov_b32 s0, 0 +; GFX1032-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1032-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1032-NEXT: v_mov_b32_e32 v5, v3 +; GFX1032-NEXT: v_mov_b32_e32 v4, v2 +; GFX1032-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 +; GFX1032-NEXT: s_cbranch_execnz .LBB12_1 +; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fsub_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b32 s14, s8 +; GFX1164-NEXT: s_add_u32 s8, s2, 44 +; GFX1164-NEXT: s_addc_u32 s9, s3, 0 +; GFX1164-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1164-NEXT: s_getpc_b64 s[4:5] +; GFX1164-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1164-NEXT: s_load_b64 s[16:17], s[4:5], 0x0 +; GFX1164-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1164-NEXT: s_mov_b32 s12, s6 +; GFX1164-NEXT: s_mov_b32 s13, s7 +; GFX1164-NEXT: s_mov_b32 s32, 0 +; GFX1164-NEXT: v_mov_b32_e32 v40, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1164-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1164-NEXT: s_mov_b64 s[0:1], 0 +; GFX1164-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1164-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1164-NEXT: v_mov_b32_e32 v5, v3 +; GFX1164-NEXT: v_mov_b32_e32 v4, v2 +; GFX1164-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[0:1] +; GFX1164-NEXT: s_cbranch_execnz .LBB12_1 +; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fsub_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_add_u32 s8, s2, 44 +; GFX1132-NEXT: s_addc_u32 s9, s3, 0 +; GFX1132-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1132-NEXT: s_getpc_b64 s[4:5] +; GFX1132-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1132-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 +; GFX1132-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1132-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v31, v0 +; GFX1132-NEXT: s_mov_b32 s12, s13 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1132-NEXT: s_mov_b32 s13, s14 +; GFX1132-NEXT: s_mov_b32 s14, s15 +; GFX1132-NEXT: s_mov_b32 s32, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1132-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1132-NEXT: s_mov_b32 s0, 0 +; GFX1132-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1132-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1132-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2 +; GFX1132-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX1132-NEXT: s_cbranch_execnz .LBB12_1 +; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s38, -1 +; GFX9-DPP-NEXT: s_mov_b32 s39, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX9-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX9-DPP-NEXT: s_mov_b32 s14, s8 +; GFX9-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX9-DPP-NEXT: s_getpc_b64 s[2:3] +; GFX9-DPP-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX9-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX9-DPP-NEXT: s_mov_b32 s12, s6 +; GFX9-DPP-NEXT: s_mov_b32 s13, s7 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX9-DPP-NEXT: s_mov_b32 s32, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX9-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX9-DPP-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX9-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB12_1 +; GFX9-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s38, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s39, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX1064-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1064-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1064-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1064-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1064-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1064-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX1064-DPP-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1064-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB12_1 +; GFX1064-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s38, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s39, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX1032-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1032-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1032-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1032-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1032-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1032-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1032-DPP-NEXT: s_mov_b32 s0, 0 +; GFX1032-DPP-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1032-DPP-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB12_1 +; GFX1032-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1164-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1164-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1164-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: s_load_b64 s[16:17], s[4:5], 0x0 +; GFX1164-DPP-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1164-DPP-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX1164-DPP-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1164-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[0:1] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB12_1 +; GFX1164-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1132-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1132-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 +; GFX1132-DPP-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v31, v0 +; GFX1132-DPP-NEXT: s_mov_b32 s12, s13 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s13, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s15 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1132-DPP-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1132-DPP-NEXT: s_mov_b32 s0, 0 +; GFX1132-DPP-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2 +; GFX1132-DPP-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB12_1 +; GFX1132-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-DPP-NEXT: s_endpgm + %divValue = call double @div.double.value() strictfp + %result = atomicrmw fsub ptr addrspace(1) %ptr, double %divValue syncscope("one-as") monotonic + ret void +} + +define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp(ptr addrspace(1) %ptr) #2{ +; GFX7LESS-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s14, -1 +; GFX7LESS-NEXT: s_mov_b32 s15, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s12, s12, s3 +; GFX7LESS-NEXT: s_addc_u32 s13, s13, 0 +; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec +; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0 +; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX7LESS-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX7LESS-NEXT: s_cbranch_execz .LBB13_3 +; GFX7LESS-NEXT: ; %bb.1: +; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[2:3] +; GFX7LESS-NEXT: s_mov_b32 s7, 0x43300000 +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0 +; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000 +; GFX7LESS-NEXT: v_add_f64 v[0:1], s[6:7], v[0:1] +; GFX7LESS-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s8 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s9 +; GFX7LESS-NEXT: s_mov_b32 s2, -1 +; GFX7LESS-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3 +; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0 +; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3] +; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5] +; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7 +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB13_2 +; GFX7LESS-NEXT: .LBB13_3: +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s10, -1 +; GFX9-NEXT: s_mov_b32 s11, 0xe00000 +; GFX9-NEXT: s_add_u32 s8, s8, s3 +; GFX9-NEXT: s_mov_b64 s[2:3], exec +; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX9-NEXT: s_addc_u32 s9, s9, 0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX9-NEXT: s_cbranch_execz .LBB13_3 +; GFX9-NEXT: ; %bb.1: +; GFX9-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX9-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX9-NEXT: s_mov_b32 s3, 0x43300000 +; GFX9-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1] +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-NEXT: s_mov_b64 s[2:3], 0 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v2, s4 +; GFX9-NEXT: v_mov_b32_e32 v3, s5 +; GFX9-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX9-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX9-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX9-NEXT: s_cbranch_execnz .LBB13_2 +; GFX9-NEXT: .LBB13_3: +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s10, -1 +; GFX1064-NEXT: s_mov_b32 s11, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s8, s8, s3 +; GFX1064-NEXT: s_mov_b64 s[2:3], exec +; GFX1064-NEXT: s_addc_u32 s9, s9, 0 +; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX1064-NEXT: s_cbranch_execz .LBB13_3 +; GFX1064-NEXT: ; %bb.1: +; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX1064-NEXT: s_mov_b32 s3, 0x43300000 +; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1064-NEXT: v_add_f64 v[0:1], 0xc3300000, s[2:3] +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 +; GFX1064-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: v_mov_b32_e32 v2, s2 +; GFX1064-NEXT: v_mov_b32_e32 v3, s3 +; GFX1064-NEXT: s_mov_b64 s[2:3], 0 +; GFX1064-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1064-NEXT: v_mov_b32_e32 v3, v1 +; GFX1064-NEXT: v_mov_b32_e32 v2, v0 +; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX1064-NEXT: s_cbranch_execnz .LBB13_2 +; GFX1064-NEXT: .LBB13_3: +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s10, -1 +; GFX1032-NEXT: s_mov_b32 s11, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s8, s8, s3 +; GFX1032-NEXT: s_mov_b32 s3, exec_lo +; GFX1032-NEXT: s_addc_u32 s9, s9, 0 +; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0 +; GFX1032-NEXT: s_mov_b32 s2, 0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo +; GFX1032-NEXT: s_cbranch_execz .LBB13_3 +; GFX1032-NEXT: ; %bb.1: +; GFX1032-NEXT: s_bcnt1_i32_b32 s4, s3 +; GFX1032-NEXT: s_mov_b32 s5, 0x43300000 +; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[4:5] +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: v_mov_b32_e32 v2, s4 +; GFX1032-NEXT: v_mov_b32_e32 v3, s5 +; GFX1032-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1032-NEXT: v_mov_b32_e32 v3, v1 +; GFX1032-NEXT: v_mov_b32_e32 v2, v0 +; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 +; GFX1032-NEXT: s_cbranch_execnz .LBB13_2 +; GFX1032-NEXT: .LBB13_3: +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_bcnt1_i32_b64 s2, exec +; GFX1164-NEXT: v_mov_b32_e32 v0, 0x43300000 +; GFX1164-NEXT: v_mov_b32_e32 v1, s2 +; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1164-NEXT: s_mov_b64 s[2:3], exec +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b32 off, v0, off offset:4 +; GFX1164-NEXT: scratch_store_b32 off, v1, off +; GFX1164-NEXT: scratch_load_b64 v[0:1], off, off +; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1164-NEXT: s_cbranch_execz .LBB13_3 +; GFX1164-NEXT: ; %bb.1: +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_load_b64 s[2:3], s[0:1], 0x0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1164-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: v_mov_b32_e32 v2, s2 +; GFX1164-NEXT: v_mov_b32_e32 v3, s3 +; GFX1164-NEXT: s_mov_b64 s[2:3], 0 +; GFX1164-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1164-NEXT: v_mov_b32_e32 v3, v1 +; GFX1164-NEXT: v_mov_b32_e32 v2, v0 +; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3] +; GFX1164-NEXT: s_cbranch_execnz .LBB13_2 +; GFX1164-NEXT: .LBB13_3: +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_bcnt1_i32_b32 s2, exec_lo +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s2 +; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1132-NEXT: s_mov_b32 s2, 0 +; GFX1132-NEXT: s_mov_b32 s3, exec_lo +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4 +; GFX1132-NEXT: scratch_store_b32 off, v1, off +; GFX1132-NEXT: scratch_load_b64 v[0:1], off, off +; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1132-NEXT: s_cbranch_execz .LBB13_3 +; GFX1132-NEXT: ; %bb.1: +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1132-NEXT: v_mov_b32_e32 v6, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_load_b64 s[4:5], s[0:1], 0x0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1132-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5 +; GFX1132-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2 +; GFX1132-NEXT: s_cbranch_execnz .LBB13_2 +; GFX1132-NEXT: .LBB13_3: +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s10, -1 +; GFX9-DPP-NEXT: s_mov_b32 s11, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s8, s8, s3 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX9-DPP-NEXT: s_addc_u32 s9, s9, 0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX9-DPP-NEXT: s_cbranch_execz .LBB13_3 +; GFX9-DPP-NEXT: ; %bb.1: +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX9-DPP-NEXT: s_mov_b32 s3, 0x43300000 +; GFX9-DPP-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1] +; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s4 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s5 +; GFX9-DPP-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB13_2 +; GFX9-DPP-NEXT: .LBB13_3: +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s10, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s11, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s8, s8, s3 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX1064-DPP-NEXT: s_addc_u32 s9, s9, 0 +; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX1064-DPP-NEXT: s_cbranch_execz .LBB13_3 +; GFX1064-DPP-NEXT: ; %bb.1: +; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX1064-DPP-NEXT: s_mov_b32 s3, 0x43300000 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1064-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[2:3] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s2 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s3 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX1064-DPP-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB13_2 +; GFX1064-DPP-NEXT: .LBB13_3: +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s10, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s11, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s8, s8, s3 +; GFX1032-DPP-NEXT: s_mov_b32 s3, exec_lo +; GFX1032-DPP-NEXT: s_addc_u32 s9, s9, 0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s2, 0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-DPP-NEXT: s_and_saveexec_b32 s4, vcc_lo +; GFX1032-DPP-NEXT: s_cbranch_execz .LBB13_3 +; GFX1032-DPP-NEXT: ; %bb.1: +; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s4, s3 +; GFX1032-DPP-NEXT: s_mov_b32 s5, 0x43300000 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[4:5] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s4 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s5 +; GFX1032-DPP-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB13_2 +; GFX1032-DPP-NEXT: .LBB13_3: +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s2, exec +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s2 +; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b32 off, v0, off offset:4 +; GFX1164-DPP-NEXT: scratch_store_b32 off, v1, off +; GFX1164-DPP-NEXT: scratch_load_b64 v[0:1], off, off +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1164-DPP-NEXT: s_cbranch_execz .LBB13_3 +; GFX1164-DPP-NEXT: ; %bb.1: +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1164-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s2 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s3 +; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX1164-DPP-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB13_2 +; GFX1164-DPP-NEXT: .LBB13_3: +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s2, exec_lo +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s2 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s2, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s3, exec_lo +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4 +; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off +; GFX1132-DPP-NEXT: scratch_load_b64 v[0:1], off, off +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1132-DPP-NEXT: s_cbranch_execz .LBB13_3 +; GFX1132-DPP-NEXT: ; %bb.1: +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5 +; GFX1132-DPP-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB13_2 +; GFX1132-DPP-NEXT: .LBB13_3: +; GFX1132-DPP-NEXT: s_endpgm + %result = atomicrmw fsub ptr addrspace(1) %ptr, double 4.0 syncscope("agent") monotonic + ret void +} + +define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_mov_b32 s32, 0 +; GFX7LESS-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s42, -1 +; GFX7LESS-NEXT: s_mov_b32 s43, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s40, s40, s9 +; GFX7LESS-NEXT: s_addc_u32 s41, s41, 0 +; GFX7LESS-NEXT: s_mov_b32 s14, s8 +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX7LESS-NEXT: s_load_dwordx2 s[36:37], s[2:3], 0x9 +; GFX7LESS-NEXT: s_mov_b32 s39, 0xf000 +; GFX7LESS-NEXT: s_mov_b32 s38, -1 +; GFX7LESS-NEXT: s_add_u32 s8, s2, 44 +; GFX7LESS-NEXT: s_addc_u32 s9, s3, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[2:3] +; GFX7LESS-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v31, v0, v2 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX7LESS-NEXT: s_mov_b32 s12, s6 +; GFX7LESS-NEXT: s_mov_b32 s13, s7 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX7LESS-NEXT: buffer_load_dwordx2 v[4:5], off, s[36:39], 0 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], 0 +; GFX7LESS-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v9, v5 +; GFX7LESS-NEXT: v_mov_b32_e32 v8, v4 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2 +; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[36:39], 0 glc +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[4:5] +; GFX7LESS-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX7LESS-NEXT: v_mov_b32_e32 v4, v6 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, v7 +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB14_1 +; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s38, -1 +; GFX9-NEXT: s_mov_b32 s39, 0xe00000 +; GFX9-NEXT: s_add_u32 s36, s36, s9 +; GFX9-NEXT: s_addc_u32 s37, s37, 0 +; GFX9-NEXT: s_mov_b32 s14, s8 +; GFX9-NEXT: s_add_u32 s8, s2, 44 +; GFX9-NEXT: s_addc_u32 s9, s3, 0 +; GFX9-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX9-NEXT: s_getpc_b64 s[2:3] +; GFX9-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX9-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX9-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX9-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX9-NEXT: s_mov_b32 s12, s6 +; GFX9-NEXT: s_mov_b32 s13, s7 +; GFX9-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX9-NEXT: s_mov_b32 s32, 0 +; GFX9-NEXT: v_mov_b32_e32 v40, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX9-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX9-NEXT: s_mov_b64 s[0:1], 0 +; GFX9-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX9-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX9-NEXT: v_mov_b32_e32 v5, v3 +; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX9-NEXT: v_mov_b32_e32 v4, v2 +; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX9-NEXT: s_cbranch_execnz .LBB14_1 +; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s38, -1 +; GFX1064-NEXT: s_mov_b32 s39, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s36, s36, s9 +; GFX1064-NEXT: s_addc_u32 s37, s37, 0 +; GFX1064-NEXT: s_mov_b32 s14, s8 +; GFX1064-NEXT: s_add_u32 s8, s2, 44 +; GFX1064-NEXT: s_addc_u32 s9, s3, 0 +; GFX1064-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1064-NEXT: s_getpc_b64 s[4:5] +; GFX1064-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1064-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1064-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1064-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1064-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1064-NEXT: s_mov_b32 s12, s6 +; GFX1064-NEXT: s_mov_b32 s13, s7 +; GFX1064-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1064-NEXT: s_mov_b32 s32, 0 +; GFX1064-NEXT: v_mov_b32_e32 v40, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1064-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1064-NEXT: s_mov_b64 s[0:1], 0 +; GFX1064-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1064-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1064-NEXT: v_mov_b32_e32 v5, v3 +; GFX1064-NEXT: v_mov_b32_e32 v4, v2 +; GFX1064-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX1064-NEXT: s_cbranch_execnz .LBB14_1 +; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s38, -1 +; GFX1032-NEXT: s_mov_b32 s39, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s36, s36, s9 +; GFX1032-NEXT: s_addc_u32 s37, s37, 0 +; GFX1032-NEXT: s_mov_b32 s14, s8 +; GFX1032-NEXT: s_add_u32 s8, s2, 44 +; GFX1032-NEXT: s_addc_u32 s9, s3, 0 +; GFX1032-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1032-NEXT: s_getpc_b64 s[4:5] +; GFX1032-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1032-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1032-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1032-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1032-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1032-NEXT: s_mov_b32 s12, s6 +; GFX1032-NEXT: s_mov_b32 s13, s7 +; GFX1032-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1032-NEXT: s_mov_b32 s32, 0 +; GFX1032-NEXT: v_mov_b32_e32 v40, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1032-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1032-NEXT: s_mov_b32 s0, 0 +; GFX1032-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1032-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1032-NEXT: v_mov_b32_e32 v5, v3 +; GFX1032-NEXT: v_mov_b32_e32 v4, v2 +; GFX1032-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 +; GFX1032-NEXT: s_cbranch_execnz .LBB14_1 +; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b32 s14, s8 +; GFX1164-NEXT: s_add_u32 s8, s2, 44 +; GFX1164-NEXT: s_addc_u32 s9, s3, 0 +; GFX1164-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1164-NEXT: s_getpc_b64 s[4:5] +; GFX1164-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1164-NEXT: s_load_b64 s[16:17], s[4:5], 0x0 +; GFX1164-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1164-NEXT: s_mov_b32 s12, s6 +; GFX1164-NEXT: s_mov_b32 s13, s7 +; GFX1164-NEXT: s_mov_b32 s32, 0 +; GFX1164-NEXT: v_mov_b32_e32 v40, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1164-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1164-NEXT: s_mov_b64 s[0:1], 0 +; GFX1164-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1164-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1164-NEXT: v_mov_b32_e32 v5, v3 +; GFX1164-NEXT: v_mov_b32_e32 v4, v2 +; GFX1164-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[0:1] +; GFX1164-NEXT: s_cbranch_execnz .LBB14_1 +; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_add_u32 s8, s2, 44 +; GFX1132-NEXT: s_addc_u32 s9, s3, 0 +; GFX1132-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1132-NEXT: s_getpc_b64 s[4:5] +; GFX1132-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1132-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 +; GFX1132-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1132-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v31, v0 +; GFX1132-NEXT: s_mov_b32 s12, s13 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1132-NEXT: s_mov_b32 s13, s14 +; GFX1132-NEXT: s_mov_b32 s14, s15 +; GFX1132-NEXT: s_mov_b32 s32, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1132-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1132-NEXT: s_mov_b32 s0, 0 +; GFX1132-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1132-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1132-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2 +; GFX1132-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX1132-NEXT: s_cbranch_execnz .LBB14_1 +; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s38, -1 +; GFX9-DPP-NEXT: s_mov_b32 s39, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX9-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX9-DPP-NEXT: s_mov_b32 s14, s8 +; GFX9-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX9-DPP-NEXT: s_getpc_b64 s[2:3] +; GFX9-DPP-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX9-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX9-DPP-NEXT: s_mov_b32 s12, s6 +; GFX9-DPP-NEXT: s_mov_b32 s13, s7 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX9-DPP-NEXT: s_mov_b32 s32, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX9-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX9-DPP-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX9-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB14_1 +; GFX9-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s38, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s39, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX1064-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1064-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1064-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1064-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1064-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1064-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX1064-DPP-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1064-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB14_1 +; GFX1064-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s38, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s39, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX1032-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1032-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1032-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1032-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1032-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1032-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1032-DPP-NEXT: s_mov_b32 s0, 0 +; GFX1032-DPP-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1032-DPP-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB14_1 +; GFX1032-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1164-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1164-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1164-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: s_load_b64 s[16:17], s[4:5], 0x0 +; GFX1164-DPP-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1164-DPP-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX1164-DPP-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1164-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[0:1] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB14_1 +; GFX1164-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1132-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1132-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 +; GFX1132-DPP-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v31, v0 +; GFX1132-DPP-NEXT: s_mov_b32 s12, s13 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s13, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s15 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1132-DPP-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1132-DPP-NEXT: s_mov_b32 s0, 0 +; GFX1132-DPP-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2 +; GFX1132-DPP-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB14_1 +; GFX1132-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-DPP-NEXT: s_endpgm + %divValue = call double @div.double.value() + %result = atomicrmw fsub ptr addrspace(1) %ptr, double %divValue syncscope("agent") monotonic + ret void +} + +define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe_structfp(ptr addrspace(1) %ptr) #1 { +; GFX7LESS-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_mov_b32 s32, 0 +; GFX7LESS-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s42, -1 +; GFX7LESS-NEXT: s_mov_b32 s43, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s40, s40, s9 +; GFX7LESS-NEXT: s_addc_u32 s41, s41, 0 +; GFX7LESS-NEXT: s_mov_b32 s14, s8 +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX7LESS-NEXT: s_load_dwordx2 s[36:37], s[2:3], 0x9 +; GFX7LESS-NEXT: s_mov_b32 s39, 0xf000 +; GFX7LESS-NEXT: s_mov_b32 s38, -1 +; GFX7LESS-NEXT: s_add_u32 s8, s2, 44 +; GFX7LESS-NEXT: s_addc_u32 s9, s3, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[2:3] +; GFX7LESS-NEXT: s_add_u32 s2, s2, div.float.value@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s3, s3, div.float.value@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v31, v0, v2 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX7LESS-NEXT: s_mov_b32 s12, s6 +; GFX7LESS-NEXT: s_mov_b32 s13, s7 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX7LESS-NEXT: buffer_load_dwordx2 v[4:5], off, s[36:39], 0 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], 0 +; GFX7LESS-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v9, v5 +; GFX7LESS-NEXT: v_mov_b32_e32 v8, v4 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2 +; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[36:39], 0 glc +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[4:5] +; GFX7LESS-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX7LESS-NEXT: v_mov_b32_e32 v4, v6 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, v7 +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB15_1 +; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s38, -1 +; GFX9-NEXT: s_mov_b32 s39, 0xe00000 +; GFX9-NEXT: s_add_u32 s36, s36, s9 +; GFX9-NEXT: s_addc_u32 s37, s37, 0 +; GFX9-NEXT: s_mov_b32 s14, s8 +; GFX9-NEXT: s_add_u32 s8, s2, 44 +; GFX9-NEXT: s_addc_u32 s9, s3, 0 +; GFX9-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX9-NEXT: s_getpc_b64 s[2:3] +; GFX9-NEXT: s_add_u32 s2, s2, div.float.value@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s3, s3, div.float.value@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX9-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX9-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX9-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX9-NEXT: s_mov_b32 s12, s6 +; GFX9-NEXT: s_mov_b32 s13, s7 +; GFX9-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX9-NEXT: s_mov_b32 s32, 0 +; GFX9-NEXT: v_mov_b32_e32 v40, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX9-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX9-NEXT: s_mov_b64 s[0:1], 0 +; GFX9-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX9-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX9-NEXT: v_mov_b32_e32 v5, v3 +; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX9-NEXT: v_mov_b32_e32 v4, v2 +; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX9-NEXT: s_cbranch_execnz .LBB15_1 +; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s38, -1 +; GFX1064-NEXT: s_mov_b32 s39, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s36, s36, s9 +; GFX1064-NEXT: s_addc_u32 s37, s37, 0 +; GFX1064-NEXT: s_mov_b32 s14, s8 +; GFX1064-NEXT: s_add_u32 s8, s2, 44 +; GFX1064-NEXT: s_addc_u32 s9, s3, 0 +; GFX1064-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1064-NEXT: s_getpc_b64 s[4:5] +; GFX1064-NEXT: s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12 +; GFX1064-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1064-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1064-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1064-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1064-NEXT: s_mov_b32 s12, s6 +; GFX1064-NEXT: s_mov_b32 s13, s7 +; GFX1064-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1064-NEXT: s_mov_b32 s32, 0 +; GFX1064-NEXT: v_mov_b32_e32 v40, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1064-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1064-NEXT: s_mov_b64 s[0:1], 0 +; GFX1064-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1064-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1064-NEXT: v_mov_b32_e32 v5, v3 +; GFX1064-NEXT: v_mov_b32_e32 v4, v2 +; GFX1064-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX1064-NEXT: s_cbranch_execnz .LBB15_1 +; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s38, -1 +; GFX1032-NEXT: s_mov_b32 s39, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s36, s36, s9 +; GFX1032-NEXT: s_addc_u32 s37, s37, 0 +; GFX1032-NEXT: s_mov_b32 s14, s8 +; GFX1032-NEXT: s_add_u32 s8, s2, 44 +; GFX1032-NEXT: s_addc_u32 s9, s3, 0 +; GFX1032-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1032-NEXT: s_getpc_b64 s[4:5] +; GFX1032-NEXT: s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12 +; GFX1032-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1032-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1032-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1032-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1032-NEXT: s_mov_b32 s12, s6 +; GFX1032-NEXT: s_mov_b32 s13, s7 +; GFX1032-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1032-NEXT: s_mov_b32 s32, 0 +; GFX1032-NEXT: v_mov_b32_e32 v40, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1032-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1032-NEXT: s_mov_b32 s0, 0 +; GFX1032-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1032-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1032-NEXT: v_mov_b32_e32 v5, v3 +; GFX1032-NEXT: v_mov_b32_e32 v4, v2 +; GFX1032-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 +; GFX1032-NEXT: s_cbranch_execnz .LBB15_1 +; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b32 s14, s8 +; GFX1164-NEXT: s_add_u32 s8, s2, 44 +; GFX1164-NEXT: s_addc_u32 s9, s3, 0 +; GFX1164-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1164-NEXT: s_getpc_b64 s[4:5] +; GFX1164-NEXT: s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12 +; GFX1164-NEXT: s_load_b64 s[16:17], s[4:5], 0x0 +; GFX1164-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1164-NEXT: s_mov_b32 s12, s6 +; GFX1164-NEXT: s_mov_b32 s13, s7 +; GFX1164-NEXT: s_mov_b32 s32, 0 +; GFX1164-NEXT: v_mov_b32_e32 v40, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1164-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1164-NEXT: s_mov_b64 s[0:1], 0 +; GFX1164-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1164-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1164-NEXT: v_mov_b32_e32 v5, v3 +; GFX1164-NEXT: v_mov_b32_e32 v4, v2 +; GFX1164-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[0:1] +; GFX1164-NEXT: s_cbranch_execnz .LBB15_1 +; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_add_u32 s8, s2, 44 +; GFX1132-NEXT: s_addc_u32 s9, s3, 0 +; GFX1132-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1132-NEXT: s_getpc_b64 s[4:5] +; GFX1132-NEXT: s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12 +; GFX1132-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 +; GFX1132-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1132-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v31, v0 +; GFX1132-NEXT: s_mov_b32 s12, s13 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1132-NEXT: s_mov_b32 s13, s14 +; GFX1132-NEXT: s_mov_b32 s14, s15 +; GFX1132-NEXT: s_mov_b32 s32, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1132-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1132-NEXT: s_mov_b32 s0, 0 +; GFX1132-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1132-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1132-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2 +; GFX1132-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX1132-NEXT: s_cbranch_execnz .LBB15_1 +; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s38, -1 +; GFX9-DPP-NEXT: s_mov_b32 s39, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX9-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX9-DPP-NEXT: s_mov_b32 s14, s8 +; GFX9-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX9-DPP-NEXT: s_getpc_b64 s[2:3] +; GFX9-DPP-NEXT: s_add_u32 s2, s2, div.float.value@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s3, s3, div.float.value@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX9-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX9-DPP-NEXT: s_mov_b32 s12, s6 +; GFX9-DPP-NEXT: s_mov_b32 s13, s7 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX9-DPP-NEXT: s_mov_b32 s32, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX9-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX9-DPP-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX9-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB15_1 +; GFX9-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s38, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s39, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX1064-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1064-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1064-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1064-DPP-NEXT: s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1064-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1064-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX1064-DPP-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1064-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB15_1 +; GFX1064-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s38, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s39, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX1032-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1032-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1032-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1032-DPP-NEXT: s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1032-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1032-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1032-DPP-NEXT: s_mov_b32 s0, 0 +; GFX1032-DPP-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1032-DPP-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB15_1 +; GFX1032-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1164-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1164-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1164-DPP-NEXT: s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: s_load_b64 s[16:17], s[4:5], 0x0 +; GFX1164-DPP-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1164-DPP-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX1164-DPP-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1164-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[0:1] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB15_1 +; GFX1164-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1132-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1132-DPP-NEXT: s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 +; GFX1132-DPP-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v31, v0 +; GFX1132-DPP-NEXT: s_mov_b32 s12, s13 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s13, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s15 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1132-DPP-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1132-DPP-NEXT: s_mov_b32 s0, 0 +; GFX1132-DPP-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2 +; GFX1132-DPP-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB15_1 +; GFX1132-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-DPP-NEXT: s_endpgm + %divValue = call double @div.float.value() strictfp + %result = atomicrmw fsub ptr addrspace(1) %ptr, double %divValue syncscope("agent") monotonic + ret void +} +define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defalut_scope_strictfp(ptr addrspace(1) %ptr) #2 { +; GFX7LESS-LABEL: global_atomic_fsub_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_movk_i32 s32, 0x800 +; GFX7LESS-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s42, -1 +; GFX7LESS-NEXT: s_mov_b32 s43, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s40, s40, s3 +; GFX7LESS-NEXT: s_addc_u32 s41, s41, 0 +; GFX7LESS-NEXT: s_mov_b32 s33, s2 +; GFX7LESS-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX7LESS-NEXT: v_mov_b32_e32 v40, v0 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec +; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s0, 0 +; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s1, v0 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX7LESS-NEXT: s_cbranch_execz .LBB16_3 +; GFX7LESS-NEXT: ; %bb.1: +; GFX7LESS-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x9 +; GFX7LESS-NEXT: s_bcnt1_i32_b64 s0, s[0:1] +; GFX7LESS-NEXT: s_mov_b32 s1, 0x43300000 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_load_dwordx2 s[2:3], s[36:37], 0x0 +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX7LESS-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1] +; GFX7LESS-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX7LESS-NEXT: s_mov_b64 s[38:39], 0 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, s2 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, s3 +; GFX7LESS-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_add_f64 v[2:3], v[0:1], -v[41:42] +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[40:43], 0 offset:4 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[40:43], 0 +; GFX7LESS-NEXT: s_add_u32 s8, s34, 44 +; GFX7LESS-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:12 +; GFX7LESS-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:8 +; GFX7LESS-NEXT: s_addc_u32 s9, s35, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX7LESS-NEXT: s_waitcnt expcnt(2) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, 0 +; GFX7LESS-NEXT: s_mov_b32 s12, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v40 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s36 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s37 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX7LESS-NEXT: v_and_b32_e32 v2, 1, v0 +; GFX7LESS-NEXT: buffer_load_dword v0, off, s[40:43], 0 +; GFX7LESS-NEXT: buffer_load_dword v1, off, s[40:43], 0 offset:4 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2 +; GFX7LESS-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB16_2 +; GFX7LESS-NEXT: .LBB16_3: +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fsub_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s42, -1 +; GFX9-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX9-NEXT: s_mov_b64 s[0:1], exec +; GFX9-NEXT: s_mov_b32 s43, 0xe00000 +; GFX9-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0 +; GFX9-NEXT: s_add_u32 s40, s40, s3 +; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0 +; GFX9-NEXT: s_addc_u32 s41, s41, 0 +; GFX9-NEXT: s_mov_b32 s33, s2 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-NEXT: s_movk_i32 s32, 0x800 +; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX9-NEXT: s_cbranch_execz .LBB16_3 +; GFX9-NEXT: ; %bb.1: +; GFX9-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1] +; GFX9-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX9-NEXT: s_mov_b32 s1, 0x43300000 +; GFX9-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1] +; GFX9-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX9-NEXT: s_mov_b64 s[38:39], 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX9-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX9-NEXT: s_add_u32 s8, s34, 44 +; GFX9-NEXT: s_addc_u32 s9, s35, 0 +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX9-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX9-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX9-NEXT: s_mov_b32 s12, s33 +; GFX9-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX9-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX9-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX9-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: v_mov_b32_e32 v2, s36 +; GFX9-NEXT: v_mov_b32_e32 v3, s37 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX9-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX9-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX9-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX9-NEXT: s_cbranch_execnz .LBB16_2 +; GFX9-NEXT: .LBB16_3: +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fsub_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s42, -1 +; GFX1064-NEXT: s_mov_b32 s43, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s40, s40, s3 +; GFX1064-NEXT: s_mov_b32 s33, s2 +; GFX1064-NEXT: s_mov_b64 s[2:3], exec +; GFX1064-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1064-NEXT: s_addc_u32 s41, s41, 0 +; GFX1064-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1064-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX1064-NEXT: s_cbranch_execz .LBB16_3 +; GFX1064-NEXT: ; %bb.1: +; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[2:3] +; GFX1064-NEXT: s_mov_b32 s1, 0x43300000 +; GFX1064-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1064-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1] +; GFX1064-NEXT: s_mov_b64 s[38:39], 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1064-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: v_mov_b32_e32 v2, s1 +; GFX1064-NEXT: v_mov_b32_e32 v1, s0 +; GFX1064-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1064-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1064-NEXT: s_mov_b32 s12, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1064-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1064-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1064-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1064-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-NEXT: v_mov_b32_e32 v2, s36 +; GFX1064-NEXT: v_mov_b32_e32 v3, s37 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1064-NEXT: s_clause 0x1 +; GFX1064-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1064-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX1064-NEXT: s_cbranch_execnz .LBB16_2 +; GFX1064-NEXT: .LBB16_3: +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fsub_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s33, s2 +; GFX1032-NEXT: s_mov_b32 s2, exec_lo +; GFX1032-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1032-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s42, -1 +; GFX1032-NEXT: s_mov_b32 s43, 0x31c16000 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-NEXT: s_add_u32 s40, s40, s3 +; GFX1032-NEXT: s_addc_u32 s41, s41, 0 +; GFX1032-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1032-NEXT: s_mov_b32 s38, 0 +; GFX1032-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo +; GFX1032-NEXT: s_cbranch_execz .LBB16_3 +; GFX1032-NEXT: ; %bb.1: +; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s2 +; GFX1032-NEXT: s_mov_b32 s1, 0x43300000 +; GFX1032-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1] +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1032-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: v_mov_b32_e32 v2, s1 +; GFX1032-NEXT: v_mov_b32_e32 v1, s0 +; GFX1032-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1032-NEXT: s_mov_b32 s12, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1032-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1032-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1032-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1032-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-NEXT: v_mov_b32_e32 v2, s36 +; GFX1032-NEXT: v_mov_b32_e32 v3, s37 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1032-NEXT: s_clause 0x1 +; GFX1032-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1032-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s38 +; GFX1032-NEXT: s_cbranch_execnz .LBB16_2 +; GFX1032-NEXT: .LBB16_3: +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fsub_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1164-NEXT: s_bcnt1_i32_b64 s0, exec +; GFX1164-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-NEXT: v_mov_b32_e32 v0, 0x43300000 +; GFX1164-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b32 off, v0, off offset:20 +; GFX1164-NEXT: scratch_store_b32 off, v1, off offset:16 +; GFX1164-NEXT: scratch_load_b64 v[0:1], off, off offset:16 +; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 +; GFX1164-NEXT: s_mov_b32 s32, 32 +; GFX1164-NEXT: s_mov_b64 s[0:1], exec +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1164-NEXT: s_cbranch_execz .LBB16_3 +; GFX1164-NEXT: ; %bb.1: +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1164-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1164-NEXT: s_mov_b32 s33, s2 +; GFX1164-NEXT: s_mov_b64 s[38:39], 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: v_mov_b32_e32 v2, s1 +; GFX1164-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-NEXT: .p2align 6 +; GFX1164-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-NEXT: s_mov_b32 s12, s33 +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-NEXT: v_mov_b32_e32 v2, s36 +; GFX1164-NEXT: v_mov_b32_e32 v3, s37 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[38:39] +; GFX1164-NEXT: s_cbranch_execnz .LBB16_2 +; GFX1164-NEXT: .LBB16_3: +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fsub_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v1, s0 +; GFX1132-NEXT: v_mov_b32_e32 v0, 0x43300000 +; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:20 +; GFX1132-NEXT: scratch_store_b32 off, v1, off offset:16 +; GFX1132-NEXT: scratch_load_b64 v[0:1], off, off offset:16 +; GFX1132-NEXT: s_mov_b32 s38, 0 +; GFX1132-NEXT: s_mov_b32 s32, 32 +; GFX1132-NEXT: s_mov_b32 s0, exec_lo +; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1132-NEXT: s_cbranch_execz .LBB16_3 +; GFX1132-NEXT: ; %bb.1: +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1132-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1132-NEXT: s_mov_b32 s33, s15 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0 +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-NEXT: .p2align 6 +; GFX1132-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-NEXT: v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-NEXT: s_mov_b32 s12, s33 +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s36 +; GFX1132-NEXT: v_dual_mov_b32 v3, s37 :: v_dual_mov_b32 v4, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38 +; GFX1132-NEXT: s_cbranch_execnz .LBB16_2 +; GFX1132-NEXT: .LBB16_3: +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s42, -1 +; GFX9-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], exec +; GFX9-DPP-NEXT: s_mov_b32 s43, 0xe00000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0 +; GFX9-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0 +; GFX9-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX9-DPP-NEXT: s_mov_b32 s33, s2 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX9-DPP-NEXT: s_cbranch_execz .LBB16_3 +; GFX9-DPP-NEXT: ; %bb.1: +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1] +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX9-DPP-NEXT: s_mov_b32 s1, 0x43300000 +; GFX9-DPP-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1] +; GFX9-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX9-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX9-DPP-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-DPP-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX9-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX9-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX9-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX9-DPP-NEXT: s_mov_b32 s12, s33 +; GFX9-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX9-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX9-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX9-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX9-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB16_2 +; GFX9-DPP-NEXT: .LBB16_3: +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s42, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s43, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX1064-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1064-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1064-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX1064-DPP-NEXT: s_cbranch_execz .LBB16_3 +; GFX1064-DPP-NEXT: ; %bb.1: +; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[2:3] +; GFX1064-DPP-NEXT: s_mov_b32 s1, 0x43300000 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1064-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1] +; GFX1064-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1064-DPP-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1064-DPP-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1064-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1064-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1064-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1064-DPP-NEXT: s_clause 0x1 +; GFX1064-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1064-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB16_2 +; GFX1064-DPP-NEXT: .LBB16_3: +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1032-DPP-NEXT: s_mov_b32 s2, exec_lo +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s42, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s43, 0x31c16000 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX1032-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1032-DPP-NEXT: s_mov_b32 s38, 0 +; GFX1032-DPP-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo +; GFX1032-DPP-NEXT: s_cbranch_execz .LBB16_3 +; GFX1032-DPP-NEXT: ; %bb.1: +; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s2 +; GFX1032-DPP-NEXT: s_mov_b32 s1, 0x43300000 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1] +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1032-DPP-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1032-DPP-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1032-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1032-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1032-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1032-DPP-NEXT: s_clause 0x1 +; GFX1032-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1032-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-DPP-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s38 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB16_2 +; GFX1032-DPP-NEXT: .LBB16_3: +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, exec +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b32 off, v0, off offset:20 +; GFX1164-DPP-NEXT: scratch_store_b32 off, v1, off offset:16 +; GFX1164-DPP-NEXT: scratch_load_b64 v[0:1], off, off offset:16 +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1164-DPP-NEXT: s_cbranch_execz .LBB16_3 +; GFX1164-DPP-NEXT: ; %bb.1: +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1164-DPP-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1164-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1164-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-DPP-NEXT: .p2align 6 +; GFX1164-DPP-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[38:39] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB16_2 +; GFX1164-DPP-NEXT: .LBB16_3: +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v1, s0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:20 +; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off offset:16 +; GFX1132-DPP-NEXT: scratch_load_b64 v[0:1], off, off offset:16 +; GFX1132-DPP-NEXT: s_mov_b32 s38, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1132-DPP-NEXT: s_cbranch_execz .LBB16_3 +; GFX1132-DPP-NEXT: ; %bb.1: +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1132-DPP-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1132-DPP-NEXT: s_mov_b32 s33, s15 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0 +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-DPP-NEXT: .p2align 6 +; GFX1132-DPP-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s36 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, s37 :: v_dual_mov_b32 v4, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-DPP-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB16_2 +; GFX1132-DPP-NEXT: .LBB16_3: +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-DPP-NEXT: s_endpgm + %result = atomicrmw fsub ptr addrspace(1) %ptr, double 4.0 monotonic, align 4 + ret void +} + +define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defalut_scope_strictfp(ptr addrspace(1) %ptr) #2 { +; GFX7LESS-LABEL: global_atomic_fsub_double_uni_address_div_value_defalut_scope_strictfp: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_movk_i32 s32, 0x800 +; GFX7LESS-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s50, -1 +; GFX7LESS-NEXT: s_mov_b32 s51, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s48, s48, s9 +; GFX7LESS-NEXT: s_addc_u32 s49, s49, 0 +; GFX7LESS-NEXT: s_mov_b32 s33, s8 +; GFX7LESS-NEXT: s_mov_b32 s40, s7 +; GFX7LESS-NEXT: s_mov_b32 s41, s6 +; GFX7LESS-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX7LESS-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX7LESS-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX7LESS-NEXT: s_load_dwordx2 s[44:45], s[2:3], 0x9 +; GFX7LESS-NEXT: s_mov_b32 s47, 0xf000 +; GFX7LESS-NEXT: s_mov_b32 s46, -1 +; GFX7LESS-NEXT: s_add_u32 s8, s36, 44 +; GFX7LESS-NEXT: s_addc_u32 s9, s37, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v42, v0, v2 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX7LESS-NEXT: s_mov_b32 s12, s41 +; GFX7LESS-NEXT: s_mov_b32 s13, s40 +; GFX7LESS-NEXT: s_mov_b32 s14, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v42 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX7LESS-NEXT: v_mov_b32_e32 v40, v0 +; GFX7LESS-NEXT: v_mov_b32_e32 v41, v1 +; GFX7LESS-NEXT: buffer_load_dwordx2 v[0:1], off, s[44:47], 0 +; GFX7LESS-NEXT: s_mov_b64 s[42:43], 0 +; GFX7LESS-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_add_f64 v[2:3], v[0:1], -v[40:41] +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:4 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[48:51], 0 +; GFX7LESS-NEXT: s_add_u32 s8, s36, 44 +; GFX7LESS-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:12 +; GFX7LESS-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:8 +; GFX7LESS-NEXT: s_addc_u32 s9, s37, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: s_waitcnt expcnt(2) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, 0 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX7LESS-NEXT: s_mov_b32 s12, s41 +; GFX7LESS-NEXT: s_mov_b32 s13, s40 +; GFX7LESS-NEXT: s_mov_b32 s14, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v42 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s44 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s45 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX7LESS-NEXT: v_and_b32_e32 v2, 1, v0 +; GFX7LESS-NEXT: buffer_load_dword v0, off, s[48:51], 0 +; GFX7LESS-NEXT: buffer_load_dword v1, off, s[48:51], 0 offset:4 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2 +; GFX7LESS-NEXT: s_or_b64 s[42:43], vcc, s[42:43] +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[42:43] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB17_1 +; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fsub_double_uni_address_div_value_defalut_scope_strictfp: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s50, -1 +; GFX9-NEXT: s_mov_b32 s51, 0xe00000 +; GFX9-NEXT: s_add_u32 s48, s48, s9 +; GFX9-NEXT: s_addc_u32 s49, s49, 0 +; GFX9-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX9-NEXT: s_mov_b32 s33, s8 +; GFX9-NEXT: s_add_u32 s8, s36, 44 +; GFX9-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX9-NEXT: s_mov_b32 s40, s7 +; GFX9-NEXT: s_mov_b32 s41, s6 +; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX9-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX9-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-NEXT: s_mov_b32 s12, s41 +; GFX9-NEXT: s_mov_b32 s13, s40 +; GFX9-NEXT: s_mov_b32 s14, s33 +; GFX9-NEXT: v_mov_b32_e32 v31, v42 +; GFX9-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-NEXT: s_movk_i32 s32, 0x800 +; GFX9-NEXT: v_mov_b32_e32 v43, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-NEXT: v_mov_b32_e32 v41, v1 +; GFX9-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX9-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-NEXT: s_mov_b64 s[44:45], 0 +; GFX9-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX9-NEXT: s_add_u32 s8, s36, 44 +; GFX9-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX9-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX9-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX9-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX9-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-NEXT: s_mov_b32 s12, s41 +; GFX9-NEXT: s_mov_b32 s13, s40 +; GFX9-NEXT: s_mov_b32 s14, s33 +; GFX9-NEXT: v_mov_b32_e32 v31, v42 +; GFX9-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: v_mov_b32_e32 v2, s42 +; GFX9-NEXT: v_mov_b32_e32 v3, s43 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX9-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX9-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX9-NEXT: s_cbranch_execnz .LBB17_1 +; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fsub_double_uni_address_div_value_defalut_scope_strictfp: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s50, -1 +; GFX1064-NEXT: s_mov_b32 s51, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s48, s48, s9 +; GFX1064-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1064-NEXT: s_addc_u32 s49, s49, 0 +; GFX1064-NEXT: s_mov_b32 s33, s8 +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1064-NEXT: s_mov_b32 s40, s7 +; GFX1064-NEXT: s_mov_b32 s41, s6 +; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1064-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1064-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX1064-NEXT: s_mov_b32 s12, s41 +; GFX1064-NEXT: s_mov_b32 s13, s40 +; GFX1064-NEXT: s_mov_b32 s14, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-NEXT: v_mov_b32_e32 v31, v42 +; GFX1064-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-NEXT: v_mov_b32_e32 v43, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-NEXT: v_mov_b32_e32 v41, v1 +; GFX1064-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX1064-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-NEXT: s_mov_b64 s[44:45], 0 +; GFX1064-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX1064-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-NEXT: v_mov_b32_e32 v31, v42 +; GFX1064-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-NEXT: v_mov_b32_e32 v2, s42 +; GFX1064-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-NEXT: s_mov_b32 s12, s41 +; GFX1064-NEXT: s_mov_b32 s13, s40 +; GFX1064-NEXT: s_mov_b32 s14, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX1064-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX1064-NEXT: v_mov_b32_e32 v3, s43 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-NEXT: s_clause 0x1 +; GFX1064-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX1064-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX1064-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX1064-NEXT: s_cbranch_execnz .LBB17_1 +; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fsub_double_uni_address_div_value_defalut_scope_strictfp: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s50, -1 +; GFX1032-NEXT: s_mov_b32 s51, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s48, s48, s9 +; GFX1032-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1032-NEXT: s_addc_u32 s49, s49, 0 +; GFX1032-NEXT: s_mov_b32 s33, s8 +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1032-NEXT: s_mov_b32 s40, s7 +; GFX1032-NEXT: s_mov_b32 s41, s6 +; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1032-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1032-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX1032-NEXT: s_mov_b32 s12, s41 +; GFX1032-NEXT: s_mov_b32 s13, s40 +; GFX1032-NEXT: s_mov_b32 s14, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-NEXT: v_mov_b32_e32 v31, v42 +; GFX1032-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-NEXT: v_mov_b32_e32 v43, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-NEXT: v_mov_b32_e32 v41, v1 +; GFX1032-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX1032-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-NEXT: s_mov_b32 s44, 0 +; GFX1032-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX1032-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-NEXT: v_mov_b32_e32 v31, v42 +; GFX1032-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-NEXT: v_mov_b32_e32 v2, s42 +; GFX1032-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-NEXT: s_mov_b32 s12, s41 +; GFX1032-NEXT: s_mov_b32 s13, s40 +; GFX1032-NEXT: s_mov_b32 s14, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX1032-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX1032-NEXT: v_mov_b32_e32 v3, s43 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-NEXT: s_clause 0x1 +; GFX1032-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX1032-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX1032-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s44 +; GFX1032-NEXT: s_cbranch_execnz .LBB17_1 +; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fsub_double_uni_address_div_value_defalut_scope_strictfp: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1164-NEXT: s_mov_b32 s33, s8 +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1164-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-NEXT: s_mov_b32 s12, s6 +; GFX1164-NEXT: s_mov_b32 s13, s7 +; GFX1164-NEXT: s_mov_b32 s14, s33 +; GFX1164-NEXT: s_mov_b32 s32, 32 +; GFX1164-NEXT: v_mov_b32_e32 v42, v0 +; GFX1164-NEXT: s_mov_b32 s40, s7 +; GFX1164-NEXT: s_mov_b32 s41, s6 +; GFX1164-NEXT: v_mov_b32_e32 v43, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: v_mov_b32_e32 v41, v1 +; GFX1164-NEXT: global_load_b64 v[1:2], v43, s[42:43] +; GFX1164-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-NEXT: s_mov_b64 s[44:45], 0 +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-NEXT: .p2align 6 +; GFX1164-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-NEXT: v_mov_b32_e32 v31, v42 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-NEXT: s_mov_b32 s12, s41 +; GFX1164-NEXT: s_mov_b32 s13, s40 +; GFX1164-NEXT: s_mov_b32 s14, s33 +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-NEXT: v_mov_b32_e32 v2, s42 +; GFX1164-NEXT: v_mov_b32_e32 v3, s43 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[44:45] +; GFX1164-NEXT: s_cbranch_execnz .LBB17_1 +; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fsub_double_uni_address_div_value_defalut_scope_strictfp: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1132-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1132-NEXT: v_mov_b32_e32 v31, v0 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1132-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1132-NEXT: s_mov_b32 s40, s14 +; GFX1132-NEXT: s_mov_b32 s41, s13 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-NEXT: s_mov_b32 s12, s13 +; GFX1132-NEXT: s_mov_b32 s13, s14 +; GFX1132-NEXT: s_mov_b32 s14, s15 +; GFX1132-NEXT: s_mov_b32 s32, 32 +; GFX1132-NEXT: s_mov_b32 s33, s15 +; GFX1132-NEXT: v_dual_mov_b32 v42, v0 :: v_dual_mov_b32 v43, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v41, v1 +; GFX1132-NEXT: global_load_b64 v[1:2], v43, s[42:43] +; GFX1132-NEXT: s_mov_b32 s44, 0 +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-NEXT: .p2align 6 +; GFX1132-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-NEXT: v_dual_mov_b32 v31, v42 :: v_dual_mov_b32 v0, 8 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-NEXT: s_mov_b32 s12, s41 +; GFX1132-NEXT: s_mov_b32 s13, s40 +; GFX1132-NEXT: s_mov_b32 s14, s33 +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s42 +; GFX1132-NEXT: v_dual_mov_b32 v3, s43 :: v_dual_mov_b32 v4, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s44 +; GFX1132-NEXT: s_cbranch_execnz .LBB17_1 +; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_defalut_scope_strictfp: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s50, -1 +; GFX9-DPP-NEXT: s_mov_b32 s51, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX9-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX9-DPP-NEXT: s_mov_b32 s33, s8 +; GFX9-DPP-NEXT: s_add_u32 s8, s36, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_mov_b32 s40, s7 +; GFX9-DPP-NEXT: s_mov_b32 s41, s6 +; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-DPP-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX9-DPP-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-DPP-NEXT: s_mov_b32 s12, s41 +; GFX9-DPP-NEXT: s_mov_b32 s13, s40 +; GFX9-DPP-NEXT: s_mov_b32 s14, s33 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX9-DPP-NEXT: v_mov_b32_e32 v43, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-DPP-NEXT: v_mov_b32_e32 v41, v1 +; GFX9-DPP-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX9-DPP-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX9-DPP-NEXT: s_add_u32 s8, s36, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX9-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-DPP-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX9-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-DPP-NEXT: s_mov_b32 s12, s41 +; GFX9-DPP-NEXT: s_mov_b32 s13, s40 +; GFX9-DPP-NEXT: s_mov_b32 s14, s33 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-DPP-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX9-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX9-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB17_1 +; GFX9-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_defalut_scope_strictfp: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s50, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s51, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX1064-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1064-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1064-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-DPP-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX1064-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1064-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v43, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v41, v1 +; GFX1064-DPP-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX1064-DPP-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX1064-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-DPP-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX1064-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-DPP-NEXT: s_clause 0x1 +; GFX1064-DPP-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX1064-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX1064-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB17_1 +; GFX1064-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_defalut_scope_strictfp: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s50, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s51, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX1032-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1032-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1032-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-DPP-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX1032-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1032-DPP-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v43, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v41, v1 +; GFX1032-DPP-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-DPP-NEXT: s_mov_b32 s44, 0 +; GFX1032-DPP-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX1032-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-DPP-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX1032-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-DPP-NEXT: s_clause 0x1 +; GFX1032-DPP-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX1032-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX1032-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-DPP-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s44 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB17_1 +; GFX1032-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_defalut_scope_strictfp: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1164-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v42, v0 +; GFX1164-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v43, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v41, v1 +; GFX1164-DPP-NEXT: global_load_b64 v[1:2], v43, s[42:43] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-DPP-NEXT: .p2align 6 +; GFX1164-DPP-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1164-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[44:45] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB17_1 +; GFX1164-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_defalut_scope_strictfp: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1132-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1132-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1132-DPP-NEXT: s_mov_b32 s40, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s41, s13 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-DPP-NEXT: s_mov_b32 s12, s13 +; GFX1132-DPP-NEXT: s_mov_b32 s13, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s15 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1132-DPP-NEXT: s_mov_b32 s33, s15 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v42, v0 :: v_dual_mov_b32 v43, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v41, v1 +; GFX1132-DPP-NEXT: global_load_b64 v[1:2], v43, s[42:43] +; GFX1132-DPP-NEXT: s_mov_b32 s44, 0 +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-DPP-NEXT: .p2align 6 +; GFX1132-DPP-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v31, v42 :: v_dual_mov_b32 v0, 8 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1132-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s42 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, s43 :: v_dual_mov_b32 v4, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-DPP-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s44 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB17_1 +; GFX1132-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-DPP-NEXT: s_endpgm + %divValue = call double @div.float.value() strictfp + %result = atomicrmw fsub ptr addrspace(1) %ptr, double %divValue monotonic, align 4 + ret void +} + attributes #0 = { "denormal-fp-math-f32"="preserve-sign,preserve-sign" "amdgpu-unsafe-fp-atomics"="true" } attributes #1 = { strictfp "denormal-fp-math-f32"="preserve-sign,preserve-sign" "amdgpu-unsafe-fp-atomics"="true" } attributes #2 = { strictfp} -- GitLab From 4e165dd5ab7f7c022e23b645cd8f4676b03a9ec4 Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Thu, 21 Mar 2024 21:39:18 -0700 Subject: [PATCH 025/306] [TableGen] Remove unused CodeGenHwModes argument from RegSizeInfo constructor. NFC --- llvm/utils/TableGen/InfoByHwMode.cpp | 4 ++-- llvm/utils/TableGen/InfoByHwMode.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/llvm/utils/TableGen/InfoByHwMode.cpp b/llvm/utils/TableGen/InfoByHwMode.cpp index 4a64421c013c..5496408bb0d3 100644 --- a/llvm/utils/TableGen/InfoByHwMode.cpp +++ b/llvm/utils/TableGen/InfoByHwMode.cpp @@ -115,7 +115,7 @@ ValueTypeByHwMode llvm::getValueTypeByHwMode(Record *Rec, return ValueTypeByHwMode(Rec, llvm::getValueType(Rec)); } -RegSizeInfo::RegSizeInfo(Record *R, const CodeGenHwModes &CGH) { +RegSizeInfo::RegSizeInfo(Record *R) { RegSize = R->getValueAsInt("RegSize"); SpillSize = R->getValueAsInt("SpillSize"); SpillAlignment = R->getValueAsInt("SpillAlignment"); @@ -139,7 +139,7 @@ void RegSizeInfo::writeToStream(raw_ostream &OS) const { RegSizeInfoByHwMode::RegSizeInfoByHwMode(Record *R, const CodeGenHwModes &CGH) { const HwModeSelect &MS = CGH.getHwModeSelect(R); for (const HwModeSelect::PairType &P : MS.Items) { - auto I = Map.insert({P.first, RegSizeInfo(P.second, CGH)}); + auto I = Map.insert({P.first, RegSizeInfo(P.second)}); assert(I.second && "Duplicate entry?"); (void)I; } diff --git a/llvm/utils/TableGen/InfoByHwMode.h b/llvm/utils/TableGen/InfoByHwMode.h index 001509e5317f..1909913c50c6 100644 --- a/llvm/utils/TableGen/InfoByHwMode.h +++ b/llvm/utils/TableGen/InfoByHwMode.h @@ -181,7 +181,7 @@ struct RegSizeInfo { unsigned SpillSize; unsigned SpillAlignment; - RegSizeInfo(Record *R, const CodeGenHwModes &CGH); + RegSizeInfo(Record *R); RegSizeInfo() = default; bool operator<(const RegSizeInfo &I) const; bool operator==(const RegSizeInfo &I) const { -- GitLab From 90454a609894ab278a87be2b9f5c49714caba8df Mon Sep 17 00:00:00 2001 From: Chen Zheng Date: Fri, 22 Mar 2024 13:23:36 +0800 Subject: [PATCH 026/306] [PowerPC][AIX] support explicit sections for -ffunction-sections (#85351) Fix crashes in https://godbolt.org/z/6voEa1o6Y --- llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp | 6 +- .../aix-xcoff-funcsect-explicitsect.ll | 142 ++++++++++++++++++ 2 files changed, 145 insertions(+), 3 deletions(-) create mode 100644 llvm/test/CodeGen/PowerPC/aix-xcoff-funcsect-explicitsect.ll diff --git a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp index 64cae1caa643..16942c6893a1 100644 --- a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp +++ b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp @@ -2845,9 +2845,9 @@ void PPCAIXAsmPrinter::emitFunctionDescriptor() { } void PPCAIXAsmPrinter::emitFunctionEntryLabel() { - // It's not necessary to emit the label when we have individual - // function in its own csect. - if (!TM.getFunctionSections()) + // For functions without user defined section, it's not necessary to emit the + // label when we have individual function in its own csect. + if (!TM.getFunctionSections() || MF->getFunction().hasSection()) PPCAsmPrinter::emitFunctionEntryLabel(); // Emit aliasing label for function entry point label. diff --git a/llvm/test/CodeGen/PowerPC/aix-xcoff-funcsect-explicitsect.ll b/llvm/test/CodeGen/PowerPC/aix-xcoff-funcsect-explicitsect.ll new file mode 100644 index 000000000000..4e94228404d6 --- /dev/null +++ b/llvm/test/CodeGen/PowerPC/aix-xcoff-funcsect-explicitsect.ll @@ -0,0 +1,142 @@ +; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc-ibm-aix-xcoff \ +; RUN: -xcoff-traceback-table=false -filetype=obj -function-sections -o %t.o < %s +; RUN: llvm-readobj -s %t.o | FileCheck %s + +define dso_local signext i32 @foo1() section "sect" { +entry: + ret i32 1 +} + +define dso_local signext i32 @foo2() section "sect2" { +entry: + ret i32 2 +} + +define dso_local signext i32 @foo3() section "sect2" { +entry: + ret i32 3 +} + +define dso_local signext i32 @foo4() { +entry: + ret i32 4 +} + +; CHECK: Symbol {{[{][[:space:]] *}}Index: [[#INDX:]]{{[[:space:]] *}}Name: sect +; CHECK-NEXT: Value (RelocatableAddress): 0x0 +; CHECK-NEXT: Section: .text +; CHECK-NEXT: Type: 0x0 +; CHECK-NEXT: StorageClass: C_HIDEXT (0x6B) +; CHECK-NEXT: NumberOfAuxEntries: 1 +; CHECK-NEXT: CSECT Auxiliary Entry { +; CHECK-NEXT: Index: [[#INDX+1]] +; CHECK-NEXT: SectionLen: 8 +; CHECK-NEXT: ParameterHashIndex: 0x0 +; CHECK-NEXT: TypeChkSectNum: 0x0 +; CHECK-NEXT: SymbolAlignmentLog2: 5 +; CHECK-NEXT: SymbolType: XTY_SD (0x1) +; CHECK-NEXT: StorageMappingClass: XMC_PR (0x0) +; CHECK-NEXT: StabInfoIndex: 0x0 +; CHECK-NEXT: StabSectNum: 0x0 +; CHECK-NEXT: } +; CHECK-NEXT: } +; CHECK-NEXT: Symbol { +; CHECK-NEXT: Index: [[#INDX+2]] +; CHECK-NEXT: Name: .foo1 +; CHECK-NEXT: Value (RelocatableAddress): 0x0 +; CHECK-NEXT: Section: .text +; CHECK-NEXT: Type: 0x0 +; CHECK-NEXT: StorageClass: C_EXT (0x2) +; CHECK-NEXT: NumberOfAuxEntries: 1 +; CHECK-NEXT: CSECT Auxiliary Entry { +; CHECK-NEXT: Index: [[#INDX+3]] +; CHECK-NEXT: ContainingCsectSymbolIndex: [[#INDX]] +; CHECK-NEXT: ParameterHashIndex: 0x0 +; CHECK-NEXT: TypeChkSectNum: 0x0 +; CHECK-NEXT: SymbolAlignmentLog2: 0 +; CHECK-NEXT: SymbolType: XTY_LD (0x2) +; CHECK-NEXT: StorageMappingClass: XMC_PR (0x0) +; CHECK-NEXT: StabInfoIndex: 0x0 +; CHECK-NEXT: StabSectNum: 0x0 +; CHECK-NEXT: } +; CHECK-NEXT: } +; CHECK-NEXT: Symbol { +; CHECK-NEXT: Index: [[#INDX+4]] +; CHECK-NEXT: Name: sect2 +; CHECK-NEXT: Value (RelocatableAddress): 0x20 +; CHECK-NEXT: Section: .text +; CHECK-NEXT: Type: 0x0 +; CHECK-NEXT: StorageClass: C_HIDEXT (0x6B) +; CHECK-NEXT: NumberOfAuxEntries: 1 +; CHECK-NEXT: CSECT Auxiliary Entry { +; CHECK-NEXT: Index: [[#INDX+5]] +; CHECK-NEXT: SectionLen: 24 +; CHECK-NEXT: ParameterHashIndex: 0x0 +; CHECK-NEXT: TypeChkSectNum: 0x0 +; CHECK-NEXT: SymbolAlignmentLog2: 5 +; CHECK-NEXT: SymbolType: XTY_SD (0x1) +; CHECK-NEXT: StorageMappingClass: XMC_PR (0x0) +; CHECK-NEXT: StabInfoIndex: 0x0 +; CHECK-NEXT: StabSectNum: 0x0 +; CHECK-NEXT: } +; CHECK-NEXT: } +; CHECK-NEXT: Symbol { +; CHECK-NEXT: Index: [[#INDX+6]] +; CHECK-NEXT: Name: .foo2 +; CHECK-NEXT: Value (RelocatableAddress): 0x20 +; CHECK-NEXT: Section: .text +; CHECK-NEXT: Type: 0x0 +; CHECK-NEXT: StorageClass: C_EXT (0x2) +; CHECK-NEXT: NumberOfAuxEntries: 1 +; CHECK-NEXT: CSECT Auxiliary Entry { +; CHECK-NEXT: Index: [[#INDX+7]] +; CHECK-NEXT: ContainingCsectSymbolIndex: [[#INDX+4]] +; CHECK-NEXT: ParameterHashIndex: 0x0 +; CHECK-NEXT: TypeChkSectNum: 0x0 +; CHECK-NEXT: SymbolAlignmentLog2: 0 +; CHECK-NEXT: SymbolType: XTY_LD (0x2) +; CHECK-NEXT: StorageMappingClass: XMC_PR (0x0) +; CHECK-NEXT: StabInfoIndex: 0x0 +; CHECK-NEXT: StabSectNum: 0x0 +; CHECK-NEXT: } +; CHECK-NEXT: } +; CHECK-NEXT: Symbol { +; CHECK-NEXT: Index: [[#INDX+8]] +; CHECK-NEXT: Name: .foo3 +; CHECK-NEXT: Value (RelocatableAddress): 0x30 +; CHECK-NEXT: Section: .text +; CHECK-NEXT: Type: 0x0 +; CHECK-NEXT: StorageClass: C_EXT (0x2) +; CHECK-NEXT: NumberOfAuxEntries: 1 +; CHECK-NEXT: CSECT Auxiliary Entry { +; CHECK-NEXT: Index: [[#INDX+9]] +; CHECK-NEXT: ContainingCsectSymbolIndex: [[#INDX+4]] +; CHECK-NEXT: ParameterHashIndex: 0x0 +; CHECK-NEXT: TypeChkSectNum: 0x0 +; CHECK-NEXT: SymbolAlignmentLog2: 0 +; CHECK-NEXT: SymbolType: XTY_LD (0x2) +; CHECK-NEXT: StorageMappingClass: XMC_PR (0x0) +; CHECK-NEXT: StabInfoIndex: 0x0 +; CHECK-NEXT: StabSectNum: 0x0 +; CHECK-NEXT: } +; CHECK-NEXT: } +; CHECK-NEXT: Symbol { +; CHECK-NEXT: Index: [[#INDX+10]] +; CHECK-NEXT: Name: .foo4 +; CHECK-NEXT: Value (RelocatableAddress): 0x40 +; CHECK-NEXT: Section: .text +; CHECK-NEXT: Type: 0x0 +; CHECK-NEXT: StorageClass: C_EXT (0x2) +; CHECK-NEXT: NumberOfAuxEntries: 1 +; CHECK-NEXT: CSECT Auxiliary Entry { +; CHECK-NEXT: Index: 16 +; CHECK-NEXT: SectionLen: 8 +; CHECK-NEXT: ParameterHashIndex: 0x0 +; CHECK-NEXT: TypeChkSectNum: 0x0 +; CHECK-NEXT: SymbolAlignmentLog2: 5 +; CHECK-NEXT: SymbolType: XTY_SD (0x1) +; CHECK-NEXT: StorageMappingClass: XMC_PR (0x0) +; CHECK-NEXT: StabInfoIndex: 0x0 +; CHECK-NEXT: StabSectNum: 0x0 +; CHECK-NEXT: } +; CHECK-NEXT: } -- GitLab From 0289ae51aa375fd297f1d03d27ff517223e5e998 Mon Sep 17 00:00:00 2001 From: Christian Ulmann Date: Fri, 22 Mar 2024 08:31:17 +0100 Subject: [PATCH 027/306] [MLIR][LLVM][SROA] Support incorrectly typed memory accesses (#85813) This commit relaxes the assumption of type consistency for LLVM dialect load and store operations in SROA. Instead, there is now a check that loads and stores are in the bounds specified by the sub-slot they access. This commit additionally removes the corresponding patterns from the type consistency pass, as they are no longer necessary. Note: It will be necessary to extend Mem2Reg with the logic for differently sized accesses as well. This is non-the-less a strict upgrade for productive flows, as the type consistency pass can produce invalid IR for some odd cases. --- mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td | 6 +- .../LLVMIR/Transforms/TypeConsistency.h | 12 -- .../mlir/Interfaces/MemorySlotInterfaces.h | 3 +- mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp | 108 ++++++++++++-- .../LLVMIR/Transforms/TypeConsistency.cpp | 101 ------------- .../Dialect/MemRef/IR/MemRefMemorySlot.cpp | 7 +- mlir/test/Dialect/LLVMIR/sroa.mlir | 90 +++++++++++ .../test/Dialect/LLVMIR/type-consistency.mlir | 140 ++---------------- 8 files changed, 204 insertions(+), 263 deletions(-) diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td index b523374f6c06..f8f9264b3889 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td @@ -323,7 +323,8 @@ def LLVM_GEPOp : LLVM_Op<"getelementptr", [Pure, } def LLVM_LoadOp : LLVM_MemAccessOpBase<"load", - [DeclareOpInterfaceMethods, + [DeclareOpInterfaceMethods, + DeclareOpInterfaceMethods, DeclareOpInterfaceMethods, DeclareOpInterfaceMethods]> { dag args = (ins LLVM_AnyPointer:$addr, @@ -402,7 +403,8 @@ def LLVM_LoadOp : LLVM_MemAccessOpBase<"load", } def LLVM_StoreOp : LLVM_MemAccessOpBase<"store", - [DeclareOpInterfaceMethods, + [DeclareOpInterfaceMethods, + DeclareOpInterfaceMethods, DeclareOpInterfaceMethods, DeclareOpInterfaceMethods]> { dag args = (ins LLVM_LoadableType:$value, diff --git a/mlir/include/mlir/Dialect/LLVMIR/Transforms/TypeConsistency.h b/mlir/include/mlir/Dialect/LLVMIR/Transforms/TypeConsistency.h index b32ac56d7079..cacb241bfd7a 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/Transforms/TypeConsistency.h +++ b/mlir/include/mlir/Dialect/LLVMIR/Transforms/TypeConsistency.h @@ -29,18 +29,6 @@ namespace LLVM { /// interpret pointee types as consistently as possible. std::unique_ptr createTypeConsistencyPass(); -/// Transforms uses of pointers to a whole struct to uses of pointers to the -/// first element of a struct. This is achieved by inserting a GEP to the first -/// element when possible. -template -class AddFieldGetterToStructDirectUse : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(User user, - PatternRewriter &rewriter) const override; -}; - /// Canonicalizes GEPs of which the base type and the pointer's type hint do not /// match. This is done by replacing the original GEP into a GEP with the type /// hint as a base type when an element of the hinted type aligns with the diff --git a/mlir/include/mlir/Interfaces/MemorySlotInterfaces.h b/mlir/include/mlir/Interfaces/MemorySlotInterfaces.h index 56e5e96aecd1..aaa261be6553 100644 --- a/mlir/include/mlir/Interfaces/MemorySlotInterfaces.h +++ b/mlir/include/mlir/Interfaces/MemorySlotInterfaces.h @@ -26,8 +26,7 @@ struct MemorySlot { /// Memory slot attached with information about its destructuring procedure. struct DestructurableMemorySlot : public MemorySlot { - /// Maps an index within the memory slot to the type of the pointer that - /// will be generated to access the element directly. + /// Maps an index within the memory slot to the corresponding subelement type. DenseMap elementPtrs; }; diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp index 00b4559658fd..0ef1d105aca6 100644 --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp @@ -13,10 +13,8 @@ #include "mlir/Dialect/LLVMIR/LLVMDialect.h" #include "mlir/Dialect/LLVMIR/LLVMTypes.h" -#include "mlir/IR/IRMapping.h" #include "mlir/IR/Matchers.h" #include "mlir/IR/PatternMatch.h" -#include "mlir/IR/ValueRange.h" #include "mlir/Interfaces/DataLayoutInterfaces.h" #include "mlir/Interfaces/MemorySlotInterfaces.h" #include "llvm/ADT/STLExtras.h" @@ -71,12 +69,8 @@ SmallVector LLVM::AllocaOp::getDestructurableSlots() { if (!destructuredType) return {}; - DenseMap allocaTypeMap; - for (Attribute index : llvm::make_first_range(destructuredType.value())) - allocaTypeMap.insert({index, LLVM::LLVMPointerType::get(getContext())}); - - return { - DestructurableMemorySlot{{getResult(), getElemType()}, {allocaTypeMap}}}; + return {DestructurableMemorySlot{{getResult(), getElemType()}, + *destructuredType}}; } DenseMap @@ -182,17 +176,107 @@ DeletionKind LLVM::StoreOp::removeBlockingUses( return DeletionKind::Delete; } +/// Checks if `slot` can be accessed through the provided access type. +static bool isValidAccessType(const MemorySlot &slot, Type accessType, + const DataLayout &dataLayout) { + return dataLayout.getTypeSize(accessType) <= + dataLayout.getTypeSize(slot.elemType); +} + LogicalResult LLVM::LoadOp::ensureOnlySafeAccesses( const MemorySlot &slot, SmallVectorImpl &mustBeSafelyUsed, const DataLayout &dataLayout) { - return success(getAddr() != slot.ptr || getType() == slot.elemType); + return success(getAddr() != slot.ptr || + isValidAccessType(slot, getType(), dataLayout)); } LogicalResult LLVM::StoreOp::ensureOnlySafeAccesses( const MemorySlot &slot, SmallVectorImpl &mustBeSafelyUsed, const DataLayout &dataLayout) { return success(getAddr() != slot.ptr || - getValue().getType() == slot.elemType); + isValidAccessType(slot, getValue().getType(), dataLayout)); +} + +/// Returns the subslot's type at the requested index. +static Type getTypeAtIndex(const DestructurableMemorySlot &slot, + Attribute index) { + auto subelementIndexMap = + slot.elemType.cast().getSubelementIndexMap(); + if (!subelementIndexMap) + return {}; + assert(!subelementIndexMap->empty()); + + // Note: Returns a null-type when no entry was found. + return subelementIndexMap->lookup(index); +} + +bool LLVM::LoadOp::canRewire(const DestructurableMemorySlot &slot, + SmallPtrSetImpl &usedIndices, + SmallVectorImpl &mustBeSafelyUsed, + const DataLayout &dataLayout) { + if (getVolatile_()) + return false; + + // A load always accesses the first element of the destructured slot. + auto index = IntegerAttr::get(IntegerType::get(getContext(), 32), 0); + Type subslotType = getTypeAtIndex(slot, index); + if (!subslotType) + return false; + + // The access can only be replaced when the subslot is read within its bounds. + if (dataLayout.getTypeSize(getType()) > dataLayout.getTypeSize(subslotType)) + return false; + + usedIndices.insert(index); + return true; +} + +DeletionKind LLVM::LoadOp::rewire(const DestructurableMemorySlot &slot, + DenseMap &subslots, + RewriterBase &rewriter, + const DataLayout &dataLayout) { + auto index = IntegerAttr::get(IntegerType::get(getContext(), 32), 0); + auto it = subslots.find(index); + assert(it != subslots.end()); + + rewriter.modifyOpInPlace( + *this, [&]() { getAddrMutable().set(it->getSecond().ptr); }); + return DeletionKind::Keep; +} + +bool LLVM::StoreOp::canRewire(const DestructurableMemorySlot &slot, + SmallPtrSetImpl &usedIndices, + SmallVectorImpl &mustBeSafelyUsed, + const DataLayout &dataLayout) { + if (getVolatile_()) + return false; + + // A store always accesses the first element of the destructured slot. + auto index = IntegerAttr::get(IntegerType::get(getContext(), 32), 0); + Type subslotType = getTypeAtIndex(slot, index); + if (!subslotType) + return false; + + // The access can only be replaced when the subslot is read within its bounds. + if (dataLayout.getTypeSize(getValue().getType()) > + dataLayout.getTypeSize(subslotType)) + return false; + + usedIndices.insert(index); + return true; +} + +DeletionKind LLVM::StoreOp::rewire(const DestructurableMemorySlot &slot, + DenseMap &subslots, + RewriterBase &rewriter, + const DataLayout &dataLayout) { + auto index = IntegerAttr::get(IntegerType::get(getContext(), 32), 0); + auto it = subslots.find(index); + assert(it != subslots.end()); + + rewriter.modifyOpInPlace( + *this, [&]() { getAddrMutable().set(it->getSecond().ptr); }); + return DeletionKind::Keep; } //===----------------------------------------------------------------------===// @@ -390,10 +474,8 @@ bool LLVM::GEPOp::canRewire(const DestructurableMemorySlot &slot, auto firstLevelIndex = dyn_cast(getIndices()[1]); if (!firstLevelIndex) return false; - assert(slot.elementPtrs.contains(firstLevelIndex)); - if (!llvm::isa(slot.elementPtrs.at(firstLevelIndex))) - return false; mustBeSafelyUsed.emplace_back({getResult(), reachedType}); + assert(slot.elementPtrs.contains(firstLevelIndex)); usedIndices.insert(firstLevelIndex); return true; } diff --git a/mlir/lib/Dialect/LLVMIR/Transforms/TypeConsistency.cpp b/mlir/lib/Dialect/LLVMIR/Transforms/TypeConsistency.cpp index b25c831bc717..3d700fe94e3b 100644 --- a/mlir/lib/Dialect/LLVMIR/Transforms/TypeConsistency.cpp +++ b/mlir/lib/Dialect/LLVMIR/Transforms/TypeConsistency.cpp @@ -49,104 +49,6 @@ static bool areBitcastCompatible(DataLayout &layout, Type lhs, Type rhs) { layout.getTypeSize(lhs) == layout.getTypeSize(rhs)); } -//===----------------------------------------------------------------------===// -// AddFieldGetterToStructDirectUse -//===----------------------------------------------------------------------===// - -/// Gets the type of the first subelement of `type` if `type` is destructurable, -/// nullptr otherwise. -static Type getFirstSubelementType(Type type) { - auto destructurable = dyn_cast(type); - if (!destructurable) - return nullptr; - - Type subelementType = destructurable.getTypeAtIndex( - IntegerAttr::get(IntegerType::get(type.getContext(), 32), 0)); - if (subelementType) - return subelementType; - - return nullptr; -} - -/// Extracts a pointer to the first field of an `elemType` from the address -/// pointer of the provided MemOp, and rewires the MemOp so it uses that pointer -/// instead. -template -static void insertFieldIndirection(MemOp op, PatternRewriter &rewriter, - Type elemType) { - PatternRewriter::InsertionGuard guard(rewriter); - - rewriter.setInsertionPointAfterValue(op.getAddr()); - SmallVector firstTypeIndices{0, 0}; - - Value properPtr = rewriter.create( - op->getLoc(), LLVM::LLVMPointerType::get(op.getContext()), elemType, - op.getAddr(), firstTypeIndices); - - rewriter.modifyOpInPlace(op, - [&]() { op.getAddrMutable().assign(properPtr); }); -} - -template <> -LogicalResult AddFieldGetterToStructDirectUse::matchAndRewrite( - LoadOp load, PatternRewriter &rewriter) const { - PatternRewriter::InsertionGuard guard(rewriter); - - Type inconsistentElementType = - isElementTypeInconsistent(load.getAddr(), load.getType()); - if (!inconsistentElementType) - return failure(); - Type firstType = getFirstSubelementType(inconsistentElementType); - if (!firstType) - return failure(); - DataLayout layout = DataLayout::closest(load); - if (!areBitcastCompatible(layout, firstType, load.getResult().getType())) - return failure(); - - insertFieldIndirection(load, rewriter, inconsistentElementType); - - // If the load does not use the first type but a type that can be casted from - // it, add a bitcast and change the load type. - if (firstType != load.getResult().getType()) { - rewriter.setInsertionPointAfterValue(load.getResult()); - BitcastOp bitcast = rewriter.create( - load->getLoc(), load.getResult().getType(), load.getResult()); - rewriter.modifyOpInPlace(load, - [&]() { load.getResult().setType(firstType); }); - rewriter.replaceAllUsesExcept(load.getResult(), bitcast.getResult(), - bitcast); - } - - return success(); -} - -template <> -LogicalResult AddFieldGetterToStructDirectUse::matchAndRewrite( - StoreOp store, PatternRewriter &rewriter) const { - PatternRewriter::InsertionGuard guard(rewriter); - - Type inconsistentElementType = - isElementTypeInconsistent(store.getAddr(), store.getValue().getType()); - if (!inconsistentElementType) - return failure(); - Type firstType = getFirstSubelementType(inconsistentElementType); - if (!firstType) - return failure(); - - DataLayout layout = DataLayout::closest(store); - // Check that the first field has the right type or can at least be bitcast - // to the right type. - if (!areBitcastCompatible(layout, firstType, store.getValue().getType())) - return failure(); - - insertFieldIndirection(store, rewriter, inconsistentElementType); - - rewriter.modifyOpInPlace( - store, [&]() { store.getValueMutable().assign(store.getValue()); }); - - return success(); -} - //===----------------------------------------------------------------------===// // CanonicalizeAlignedGep //===----------------------------------------------------------------------===// @@ -684,9 +586,6 @@ struct LLVMTypeConsistencyPass : public LLVM::impl::LLVMTypeConsistencyBase { void runOnOperation() override { RewritePatternSet rewritePatterns(&getContext()); - rewritePatterns.add>(&getContext()); - rewritePatterns.add>( - &getContext()); rewritePatterns.add(&getContext()); rewritePatterns.add(&getContext(), maxVectorSplitSize); rewritePatterns.add(&getContext()); diff --git a/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp b/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp index 7be4056fb2fc..6c5250d527ad 100644 --- a/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp +++ b/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp @@ -120,11 +120,8 @@ memref::AllocaOp::getDestructurableSlots() { if (!destructuredType) return {}; - DenseMap indexMap; - for (auto const &[index, type] : *destructuredType) - indexMap.insert({index, MemRefType::get({}, type)}); - - return {DestructurableMemorySlot{{getMemref(), memrefType}, indexMap}}; + return { + DestructurableMemorySlot{{getMemref(), memrefType}, *destructuredType}}; } DenseMap diff --git a/mlir/test/Dialect/LLVMIR/sroa.mlir b/mlir/test/Dialect/LLVMIR/sroa.mlir index 02d25f27f978..ca49b1298b0e 100644 --- a/mlir/test/Dialect/LLVMIR/sroa.mlir +++ b/mlir/test/Dialect/LLVMIR/sroa.mlir @@ -215,3 +215,93 @@ llvm.func @no_nested_dynamic_indexing(%arg: i32) -> i32 { // CHECK: llvm.return %[[RES]] : i32 llvm.return %3 : i32 } + +// ----- + +// CHECK-LABEL: llvm.func @store_first_field +llvm.func @store_first_field(%arg: i32) { + %0 = llvm.mlir.constant(1 : i32) : i32 + // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x i32 + %1 = llvm.alloca %0 x !llvm.struct<"foo", (i32, i32, i32)> : (i32) -> !llvm.ptr + // CHECK-NEXT: llvm.store %{{.*}}, %[[ALLOCA]] : i32 + llvm.store %arg, %1 : i32, !llvm.ptr + llvm.return +} + +// ----- + +// CHECK-LABEL: llvm.func @store_first_field_different_type +// CHECK-SAME: (%[[ARG:.*]]: f32) +llvm.func @store_first_field_different_type(%arg: f32) { + %0 = llvm.mlir.constant(1 : i32) : i32 + // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x i32 + %1 = llvm.alloca %0 x !llvm.struct<"foo", (i32, i32, i32)> : (i32) -> !llvm.ptr + // CHECK-NEXT: llvm.store %[[ARG]], %[[ALLOCA]] : f32 + llvm.store %arg, %1 : f32, !llvm.ptr + llvm.return +} + +// ----- + +// CHECK-LABEL: llvm.func @store_sub_field +// CHECK-SAME: (%[[ARG:.*]]: f32) +llvm.func @store_sub_field(%arg: f32) { + %0 = llvm.mlir.constant(1 : i32) : i32 + // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x i64 + %1 = llvm.alloca %0 x !llvm.struct<"foo", (i64, i32)> : (i32) -> !llvm.ptr + // CHECK-NEXT: llvm.store %[[ARG]], %[[ALLOCA]] : f32 + llvm.store %arg, %1 : f32, !llvm.ptr + llvm.return +} + +// ----- + +// CHECK-LABEL: llvm.func @load_first_field +llvm.func @load_first_field() -> i32 { + %0 = llvm.mlir.constant(1 : i32) : i32 + // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x i32 + %1 = llvm.alloca %0 x !llvm.struct<"foo", (i32, i32, i32)> : (i32) -> !llvm.ptr + // CHECK-NEXT: %[[RES:.*]] = llvm.load %[[ALLOCA]] : !llvm.ptr -> i32 + %2 = llvm.load %1 : !llvm.ptr -> i32 + // CHECK: llvm.return %[[RES]] : i32 + llvm.return %2 : i32 +} + +// ----- + +// CHECK-LABEL: llvm.func @load_first_field_different_type +llvm.func @load_first_field_different_type() -> f32 { + %0 = llvm.mlir.constant(1 : i32) : i32 + // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x i32 + %1 = llvm.alloca %0 x !llvm.struct<"foo", (i32, i32, i32)> : (i32) -> !llvm.ptr + // CHECK-NEXT: %[[RES:.*]] = llvm.load %[[ALLOCA]] : !llvm.ptr -> f32 + %2 = llvm.load %1 : !llvm.ptr -> f32 + // CHECK: llvm.return %[[RES]] : f32 + llvm.return %2 : f32 +} + +// ----- + +// CHECK-LABEL: llvm.func @load_sub_field +llvm.func @load_sub_field() -> i32 { + %0 = llvm.mlir.constant(1 : i32) : i32 + // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x i64 : (i32) -> !llvm.ptr + %1 = llvm.alloca %0 x !llvm.struct<(i64, i32)> : (i32) -> !llvm.ptr + // CHECK-NEXT: %[[RES:.*]] = llvm.load %[[ALLOCA]] + %res = llvm.load %1 : !llvm.ptr -> i32 + // CHECK: llvm.return %[[RES]] : i32 + llvm.return %res : i32 +} + +// ----- + +// CHECK-LABEL: llvm.func @vector_store_type_mismatch +// CHECK-SAME: %[[ARG:.*]]: vector<4xi32> +llvm.func @vector_store_type_mismatch(%arg: vector<4xi32>) { + %0 = llvm.mlir.constant(1 : i32) : i32 + // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x vector<4xf32> + %1 = llvm.alloca %0 x !llvm.struct<"foo", (vector<4xf32>)> : (i32) -> !llvm.ptr + // CHECK-NEXT: llvm.store %[[ARG]], %[[ALLOCA]] + llvm.store %arg, %1 : vector<4xi32>, !llvm.ptr + llvm.return +} diff --git a/mlir/test/Dialect/LLVMIR/type-consistency.mlir b/mlir/test/Dialect/LLVMIR/type-consistency.mlir index 021151b929d8..a6176142f174 100644 --- a/mlir/test/Dialect/LLVMIR/type-consistency.mlir +++ b/mlir/test/Dialect/LLVMIR/type-consistency.mlir @@ -26,63 +26,6 @@ llvm.func @same_address_keep_inbounds(%arg: i32) { // ----- -// CHECK-LABEL: llvm.func @struct_store_instead_of_first_field -llvm.func @struct_store_instead_of_first_field(%arg: i32) { - %0 = llvm.mlir.constant(1 : i32) : i32 - // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (i32, i32, i32)> - %1 = llvm.alloca %0 x !llvm.struct<"foo", (i32, i32, i32)> : (i32) -> !llvm.ptr - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i32, i32, i32)> - // CHECK: llvm.store %{{.*}}, %[[GEP]] : i32 - llvm.store %arg, %1 : i32, !llvm.ptr - llvm.return -} - -// ----- - -// CHECK-LABEL: llvm.func @struct_store_instead_of_first_field_same_size -// CHECK-SAME: (%[[ARG:.*]]: f32) -llvm.func @struct_store_instead_of_first_field_same_size(%arg: f32) { - %0 = llvm.mlir.constant(1 : i32) : i32 - // CHECK-DAG: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (i32, i32, i32)> - %1 = llvm.alloca %0 x !llvm.struct<"foo", (i32, i32, i32)> : (i32) -> !llvm.ptr - // CHECK-DAG: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i32, i32, i32)> - // CHECK-DAG: %[[BITCAST:.*]] = llvm.bitcast %[[ARG]] : f32 to i32 - // CHECK: llvm.store %[[BITCAST]], %[[GEP]] : i32 - llvm.store %arg, %1 : f32, !llvm.ptr - llvm.return -} - -// ----- - -// CHECK-LABEL: llvm.func @struct_load_instead_of_first_field -llvm.func @struct_load_instead_of_first_field() -> i32 { - %0 = llvm.mlir.constant(1 : i32) : i32 - // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (i32, i32, i32)> - %1 = llvm.alloca %0 x !llvm.struct<"foo", (i32, i32, i32)> : (i32) -> !llvm.ptr - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i32, i32, i32)> - // CHECK: %[[RES:.*]] = llvm.load %[[GEP]] : !llvm.ptr -> i32 - %2 = llvm.load %1 : !llvm.ptr -> i32 - // CHECK: llvm.return %[[RES]] : i32 - llvm.return %2 : i32 -} - -// ----- - -// CHECK-LABEL: llvm.func @struct_load_instead_of_first_field_same_size -llvm.func @struct_load_instead_of_first_field_same_size() -> f32 { - %0 = llvm.mlir.constant(1 : i32) : i32 - // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (i32, i32, i32)> - %1 = llvm.alloca %0 x !llvm.struct<"foo", (i32, i32, i32)> : (i32) -> !llvm.ptr - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i32, i32, i32)> - // CHECK: %[[LOADED:.*]] = llvm.load %[[GEP]] : !llvm.ptr -> i32 - // CHECK: %[[RES:.*]] = llvm.bitcast %[[LOADED]] : i32 to f32 - %2 = llvm.load %1 : !llvm.ptr -> f32 - // CHECK: llvm.return %[[RES]] : f32 - llvm.return %2 : f32 -} - -// ----- - // CHECK-LABEL: llvm.func @index_in_final_padding llvm.func @index_in_final_padding(%arg: i32) { %0 = llvm.mlir.constant(1 : i32) : i32 @@ -135,22 +78,6 @@ llvm.func @index_not_in_padding_because_packed(%arg: i16) { // ----- -// CHECK-LABEL: llvm.func @index_to_struct -// CHECK-SAME: (%[[ARG:.*]]: i32) -llvm.func @index_to_struct(%arg: i32) { - %0 = llvm.mlir.constant(1 : i32) : i32 - // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (i32, struct<"bar", (i32, i32)>)> - %1 = llvm.alloca %0 x !llvm.struct<"foo", (i32, struct<"bar", (i32, i32)>)> : (i32) -> !llvm.ptr - // CHECK: %[[GEP0:.*]] = llvm.getelementptr %[[ALLOCA]][0, 1] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i32, struct<"bar", (i32, i32)>)> - // CHECK: %[[GEP1:.*]] = llvm.getelementptr %[[GEP0]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"bar", (i32, i32)> - %7 = llvm.getelementptr %1[4] : (!llvm.ptr) -> !llvm.ptr, i8 - // CHECK: llvm.store %[[ARG]], %[[GEP1]] - llvm.store %arg, %7 : i32, !llvm.ptr - llvm.return -} - -// ----- - // CHECK-LABEL: llvm.func @no_crash_on_negative_gep_index llvm.func @no_crash_on_negative_gep_index() { %0 = llvm.mlir.constant(1.000000e+00 : f16) : f16 @@ -175,10 +102,9 @@ llvm.func @coalesced_store_ints(%arg: i64) { // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (i32, i32)> %1 = llvm.alloca %0 x !llvm.struct<"foo", (i32, i32)> : (i32) -> !llvm.ptr - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i32, i32)> // CHECK: %[[SHR:.*]] = llvm.lshr %[[ARG]], %[[CST0]] // CHECK: %[[TRUNC:.*]] = llvm.trunc %[[SHR]] : i64 to i32 - // CHECK: llvm.store %[[TRUNC]], %[[GEP]] + // CHECK: llvm.store %[[TRUNC]], %[[ALLOCA]] // CHECK: %[[SHR:.*]] = llvm.lshr %[[ARG]], %[[CST32]] : i64 // CHECK: %[[TRUNC:.*]] = llvm.trunc %[[SHR]] : i64 to i32 // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 1] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i32, i32)> @@ -225,11 +151,9 @@ llvm.func @coalesced_store_floats(%arg: i64) { // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (f32, f32)> %1 = llvm.alloca %0 x !llvm.struct<"foo", (f32, f32)> : (i32) -> !llvm.ptr - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (f32, f32)> // CHECK: %[[SHR:.*]] = llvm.lshr %[[ARG]], %[[CST0]] // CHECK: %[[TRUNC:.*]] = llvm.trunc %[[SHR]] : i64 to i32 - // CHECK: %[[BIT_CAST:.*]] = llvm.bitcast %[[TRUNC]] : i32 to f32 - // CHECK: llvm.store %[[BIT_CAST]], %[[GEP]] + // CHECK: llvm.store %[[TRUNC]], %[[ALLOCA]] // CHECK: %[[SHR:.*]] = llvm.lshr %[[ARG]], %[[CST32]] : i64 // CHECK: %[[TRUNC:.*]] = llvm.trunc %[[SHR]] : i64 to i32 // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 1] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (f32, f32)> @@ -298,10 +222,9 @@ llvm.func @coalesced_store_packed_struct(%arg: i64) { // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", packed (i16, i32, i16)> %1 = llvm.alloca %0 x !llvm.struct<"foo", packed (i16, i32, i16)> : (i32) -> !llvm.ptr - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", packed (i16, i32, i16)> // CHECK: %[[SHR:.*]] = llvm.lshr %[[ARG]], %[[CST0]] // CHECK: %[[TRUNC:.*]] = llvm.trunc %[[SHR]] : i64 to i16 - // CHECK: llvm.store %[[TRUNC]], %[[GEP]] + // CHECK: llvm.store %[[TRUNC]], %[[ALLOCA]] // CHECK: %[[SHR:.*]] = llvm.lshr %[[ARG]], %[[CST16]] // CHECK: %[[TRUNC:.*]] = llvm.trunc %[[SHR]] : i64 to i32 // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 1] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", packed (i16, i32, i16)> @@ -328,9 +251,8 @@ llvm.func @vector_write_split(%arg: vector<4xi32>) { // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (i32, i32, i32, i32)> %1 = llvm.alloca %0 x !llvm.struct<"foo", (i32, i32, i32, i32)> : (i32) -> !llvm.ptr - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i32, i32, i32, i32)> // CHECK: %[[EXTRACT:.*]] = llvm.extractelement %[[ARG]][%[[CST0]] : i32] : vector<4xi32> - // CHECK: llvm.store %[[EXTRACT]], %[[GEP]] : i32, !llvm.ptr + // CHECK: llvm.store %[[EXTRACT]], %[[ALLOCA]] : i32, !llvm.ptr // CHECK: %[[EXTRACT:.*]] = llvm.extractelement %[[ARG]][%[[CST1]] : i32] : vector<4xi32> // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 1] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i32, i32, i32, i32)> @@ -405,36 +327,6 @@ llvm.func @vector_write_split_struct(%arg: vector<2xi64>) { // ----- -// CHECK-LABEL: llvm.func @type_consistent_vector_store -// CHECK-SAME: %[[ARG:.*]]: vector<4xi32> -llvm.func @type_consistent_vector_store(%arg: vector<4xi32>) { - %0 = llvm.mlir.constant(1 : i32) : i32 - // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (vector<4xi32>)> - %1 = llvm.alloca %0 x !llvm.struct<"foo", (vector<4xi32>)> : (i32) -> !llvm.ptr - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (vector<4xi32>)> - // CHECK: llvm.store %[[ARG]], %[[GEP]] - llvm.store %arg, %1 : vector<4xi32>, !llvm.ptr - llvm.return -} - -// ----- - -// CHECK-LABEL: llvm.func @type_consistent_vector_store_other_type -// CHECK-SAME: %[[ARG:.*]]: vector<4xi32> -llvm.func @type_consistent_vector_store_other_type(%arg: vector<4xi32>) { - %0 = llvm.mlir.constant(1 : i32) : i32 - // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (vector<4xf32>)> - %1 = llvm.alloca %0 x !llvm.struct<"foo", (vector<4xf32>)> : (i32) -> !llvm.ptr - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (vector<4xf32>)> - // CHECK: %[[BIT_CAST:.*]] = llvm.bitcast %[[ARG]] : vector<4xi32> to vector<4xf32> - // CHECK: llvm.store %[[BIT_CAST]], %[[GEP]] - llvm.store %arg, %1 : vector<4xi32>, !llvm.ptr - // CHECK-NOT: llvm.store %[[ARG]], %[[ALLOCA]] - llvm.return -} - -// ----- - // CHECK-LABEL: llvm.func @bitcast_insertion // CHECK-SAME: %[[ARG:.*]]: i32 llvm.func @bitcast_insertion(%arg: i32) { @@ -478,10 +370,9 @@ llvm.func @coalesced_store_ints_subaggregate(%arg: i64) { %3 = llvm.getelementptr %1[0, 1, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i64, struct<(i32, i32)>)> // CHECK: %[[TOP_GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 1] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i64, struct<(i32, i32)>)> - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[TOP_GEP]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(i32, i32)> // CHECK: %[[SHR:.*]] = llvm.lshr %[[ARG]], %[[CST0]] // CHECK: %[[TRUNC:.*]] = llvm.trunc %[[SHR]] : i64 to i32 - // CHECK: llvm.store %[[TRUNC]], %[[GEP]] + // CHECK: llvm.store %[[TRUNC]], %[[TOP_GEP]] // CHECK: %[[SHR:.*]] = llvm.lshr %[[ARG]], %[[CST32]] : i64 // CHECK: %[[TRUNC:.*]] = llvm.trunc %[[SHR]] : i64 to i32 // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[TOP_GEP]][0, 1] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(i32, i32)> @@ -520,10 +411,9 @@ llvm.func @overlapping_int_aggregate_store(%arg: i64) { // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (i16, struct<(i16, i16, i16)>)> %1 = llvm.alloca %0 x !llvm.struct<"foo", (i16, struct<(i16, i16, i16)>)> : (i32) -> !llvm.ptr - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i16, struct<(i16, i16, i16)>)> // CHECK: %[[SHR:.*]] = llvm.lshr %[[ARG]], %[[CST0]] // CHECK: %[[TRUNC:.*]] = llvm.trunc %[[SHR]] : i64 to i16 - // CHECK: llvm.store %[[TRUNC]], %[[GEP]] + // CHECK: llvm.store %[[TRUNC]], %[[ALLOCA]] // CHECK: %[[SHR:.*]] = llvm.lshr %[[ARG]], %[[CST16]] : i64 // CHECK: [[TRUNC:.*]] = llvm.trunc %[[SHR]] : i64 to i48 @@ -531,8 +421,7 @@ llvm.func @overlapping_int_aggregate_store(%arg: i64) { // Normal integer splitting of [[TRUNC]] follows: - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[TOP_GEP]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(i16, i16, i16)> - // CHECK: llvm.store %{{.*}}, %[[GEP]] + // CHECK: llvm.store %{{.*}}, %[[TOP_GEP]] // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[TOP_GEP]][0, 1] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(i16, i16, i16)> // CHECK: llvm.store %{{.*}}, %[[GEP]] // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[TOP_GEP]][0, 2] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(i16, i16, i16)> @@ -557,14 +446,12 @@ llvm.func @overlapping_vector_aggregate_store(%arg: vector<4 x i16>) { // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (i16, struct<(i16, i16, i16)>)> %1 = llvm.alloca %0 x !llvm.struct<"foo", (i16, struct<(i16, i16, i16)>)> : (i32) -> !llvm.ptr - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i16, struct<(i16, i16, i16)>)> // CHECK: %[[EXTRACT:.*]] = llvm.extractelement %[[ARG]][%[[CST0]] : i32] - // CHECK: llvm.store %[[EXTRACT]], %[[GEP]] + // CHECK: llvm.store %[[EXTRACT]], %[[ALLOCA]] // CHECK: %[[EXTRACT:.*]] = llvm.extractelement %[[ARG]][%[[CST1]] : i32] // CHECK: %[[GEP0:.*]] = llvm.getelementptr %[[ALLOCA]][0, 1] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i16, struct<(i16, i16, i16)>)> - // CHECK: %[[GEP1:.*]] = llvm.getelementptr %[[GEP0]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(i16, i16, i16)> - // CHECK: llvm.store %[[EXTRACT]], %[[GEP1]] + // CHECK: llvm.store %[[EXTRACT]], %[[GEP0]] // CHECK: %[[EXTRACT:.*]] = llvm.extractelement %[[ARG]][%[[CST2]] : i32] // CHECK: %[[GEP0:.*]] = llvm.getelementptr %[[ALLOCA]][0, 1] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i16, struct<(i16, i16, i16)>)> @@ -593,10 +480,9 @@ llvm.func @partially_overlapping_aggregate_store(%arg: i64) { // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (i16, struct<(i16, i16, i16, i16)>)> %1 = llvm.alloca %0 x !llvm.struct<"foo", (i16, struct<(i16, i16, i16, i16)>)> : (i32) -> !llvm.ptr - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i16, struct<(i16, i16, i16, i16)>)> // CHECK: %[[SHR:.*]] = llvm.lshr %[[ARG]], %[[CST0]] // CHECK: %[[TRUNC:.*]] = llvm.trunc %[[SHR]] : i64 to i16 - // CHECK: llvm.store %[[TRUNC]], %[[GEP]] + // CHECK: llvm.store %[[TRUNC]], %[[ALLOCA]] // CHECK: %[[SHR:.*]] = llvm.lshr %[[ARG]], %[[CST16]] : i64 // CHECK: [[TRUNC:.*]] = llvm.trunc %[[SHR]] : i64 to i48 @@ -604,8 +490,7 @@ llvm.func @partially_overlapping_aggregate_store(%arg: i64) { // Normal integer splitting of [[TRUNC]] follows: - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[TOP_GEP]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(i16, i16, i16, i16)> - // CHECK: llvm.store %{{.*}}, %[[GEP]] + // CHECK: llvm.store %{{.*}}, %[[TOP_GEP]] // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[TOP_GEP]][0, 1] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(i16, i16, i16, i16)> // CHECK: llvm.store %{{.*}}, %[[GEP]] // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[TOP_GEP]][0, 2] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(i16, i16, i16, i16)> @@ -651,10 +536,9 @@ llvm.func @coalesced_store_ints_array(%arg: i64) { // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.array<2 x i32> %1 = llvm.alloca %0 x !llvm.array<2 x i32> : (i32) -> !llvm.ptr - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.array<2 x i32> // CHECK: %[[SHR:.*]] = llvm.lshr %[[ARG]], %[[CST0]] // CHECK: %[[TRUNC:.*]] = llvm.trunc %[[SHR]] : i64 to i32 - // CHECK: llvm.store %[[TRUNC]], %[[GEP]] + // CHECK: llvm.store %[[TRUNC]], %[[ALLOCA]] // CHECK: %[[SHR:.*]] = llvm.lshr %[[ARG]], %[[CST32]] : i64 // CHECK: %[[TRUNC:.*]] = llvm.trunc %[[SHR]] : i64 to i32 // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 1] : (!llvm.ptr) -> !llvm.ptr, !llvm.array<2 x i32> -- GitLab From c5f839bd58e7f888acc4cb39a18e9e5bbaa9fb0a Mon Sep 17 00:00:00 2001 From: Guillaume Chatelet Date: Fri, 22 Mar 2024 09:50:42 +0100 Subject: [PATCH 028/306] [reland][libc] Add reverse_iterator comparisons (#86188) This is a reland of #86147 but with a proper `base()` function. https://en.cppreference.com/w/cpp/iterator/reverse_iterator/operator_cmp --- libc/src/__support/CPP/iterator.h | 35 +++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/libc/src/__support/CPP/iterator.h b/libc/src/__support/CPP/iterator.h index c5bfb1912c7b..b0fd5c9f22ae 100644 --- a/libc/src/__support/CPP/iterator.h +++ b/libc/src/__support/CPP/iterator.h @@ -20,6 +20,7 @@ namespace cpp { template struct iterator_traits; template struct iterator_traits { using reference = T &; + using value_type = T; }; template class reverse_iterator { @@ -27,6 +28,8 @@ template class reverse_iterator { public: using reference = typename iterator_traits::reference; + using value_type = typename iterator_traits::value_type; + using iterator_type = Iter; LIBC_INLINE reverse_iterator() : current() {} LIBC_INLINE constexpr explicit reverse_iterator(Iter it) : current(it) {} @@ -38,6 +41,38 @@ public: LIBC_INLINE constexpr explicit reverse_iterator(const Other &it) : current(it) {} + LIBC_INLINE friend constexpr bool operator==(const reverse_iterator &lhs, + const reverse_iterator &rhs) { + return lhs.base() == rhs.base(); + } + + LIBC_INLINE friend constexpr bool operator!=(const reverse_iterator &lhs, + const reverse_iterator &rhs) { + return lhs.base() != rhs.base(); + } + + LIBC_INLINE friend constexpr bool operator<(const reverse_iterator &lhs, + const reverse_iterator &rhs) { + return lhs.base() > rhs.base(); + } + + LIBC_INLINE friend constexpr bool operator<=(const reverse_iterator &lhs, + const reverse_iterator &rhs) { + return lhs.base() >= rhs.base(); + } + + LIBC_INLINE friend constexpr bool operator>(const reverse_iterator &lhs, + const reverse_iterator &rhs) { + return lhs.base() < rhs.base(); + } + + LIBC_INLINE friend constexpr bool operator>=(const reverse_iterator &lhs, + const reverse_iterator &rhs) { + return lhs.base() <= rhs.base(); + } + + LIBC_INLINE constexpr iterator_type base() const { return current; } + LIBC_INLINE constexpr reference operator*() const { Iter tmp = current; return *--tmp; -- GitLab From 5f1b2cffe5fab0aa733fc8d5f1546c1c800faac4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20Warzy=C5=84ski?= Date: Fri, 22 Mar 2024 09:37:43 +0000 Subject: [PATCH 029/306] [mlir][vector] Add support for masks in castAwayContractionLeadingOneDim (#81906) Updates `castAwayContractionLeadingOneDim` to inherit from `MaskableOpRewritePattern` so that this pattern can support masking. Builds on top of #83827 --- .../Vector/Transforms/VectorTransforms.h | 6 +- .../mlir/Dialect/Vector/Utils/VectorUtils.h | 10 +- .../Transforms/VectorDropLeadUnitDim.cpp | 50 +++++---- .../vector-dropleadunitdim-transforms.mlir | 104 +++++++++++++----- 4 files changed, 114 insertions(+), 56 deletions(-) diff --git a/mlir/include/mlir/Dialect/Vector/Transforms/VectorTransforms.h b/mlir/include/mlir/Dialect/Vector/Transforms/VectorTransforms.h index 08d3bb157a0e..1f7d6411cd5a 100644 --- a/mlir/include/mlir/Dialect/Vector/Transforms/VectorTransforms.h +++ b/mlir/include/mlir/Dialect/Vector/Transforms/VectorTransforms.h @@ -110,8 +110,10 @@ void transferOpflowOpt(RewriterBase &rewriter, Operation *rootOp); /// Cast away the leading unit dim, if exists, for the given contract op. /// Return success if the transformation applies; return failure otherwise. -LogicalResult castAwayContractionLeadingOneDim(vector::ContractionOp contractOp, - RewriterBase &rewriter); +FailureOr +castAwayContractionLeadingOneDim(vector::ContractionOp contractOp, + MaskingOpInterface maskingOp, + RewriterBase &rewriter); } // namespace vector } // namespace mlir diff --git a/mlir/include/mlir/Dialect/Vector/Utils/VectorUtils.h b/mlir/include/mlir/Dialect/Vector/Utils/VectorUtils.h index 35e76a8b623a..2c548fb67402 100644 --- a/mlir/include/mlir/Dialect/Vector/Utils/VectorUtils.h +++ b/mlir/include/mlir/Dialect/Vector/Utils/VectorUtils.h @@ -127,8 +127,8 @@ SmallVector getMixedSizesXfer(bool hasTensorSemantics, /// responsible for providing an updated ("rewritten") version of: /// a. the source Op when mask _is not_ present, /// b. the source Op and the masking Op when mask _is_ present. -/// Note that the return value from `matchAndRewriteMaskableOp` depends on the -/// case above. +/// To use this pattern, implement `matchAndRewriteMaskableOp`. Note that +/// the return value will depend on the case above. template struct MaskableOpRewritePattern : OpRewritePattern { using OpRewritePattern::OpRewritePattern; @@ -162,9 +162,9 @@ private: } public: - // Matches SourceOp that can potentially be masked with `maskingOp`. If the - // latter is present, returns an updated masking op (with a replacement for - // `sourceOp` nested inside). Otherwise, returns an updated `sourceOp`. + // Matches `sourceOp` that can potentially be masked with `maskingOp`. If the + // latter is present, returns a replacement for `maskingOp`. Otherwise, + // returns a replacement for `sourceOp`. virtual FailureOr matchAndRewriteMaskableOp(SourceOp sourceOp, MaskingOpInterface maskingOp, PatternRewriter &rewriter) const = 0; diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp index 74382b027c2f..593c1e53557a 100644 --- a/mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp +++ b/mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp @@ -329,12 +329,10 @@ struct CastAwayTransferWriteLeadingOneDim } // namespace -LogicalResult +FailureOr mlir::vector::castAwayContractionLeadingOneDim(vector::ContractionOp contractOp, + MaskingOpInterface maskingOp, RewriterBase &rewriter) { - // TODO(#78787): Not supported masked op yet. - if (cast(contractOp.getOperation()).isMasked()) - return failure(); VectorType oldAccType = dyn_cast(contractOp.getAccType()); if (oldAccType == nullptr) return failure(); @@ -368,6 +366,7 @@ mlir::vector::castAwayContractionLeadingOneDim(vector::ContractionOp contractOp, SmallVector operands = {contractOp.getLhs(), contractOp.getRhs(), contractOp.getAcc()}; SmallVector newOperands; + auto loc = contractOp.getLoc(); for (const auto &it : llvm::enumerate(oldIndexingMaps)) { // Check if the dim to be dropped exists as a leading dim in the operand @@ -405,7 +404,7 @@ mlir::vector::castAwayContractionLeadingOneDim(vector::ContractionOp contractOp, map = AffineMap::get(map.getNumDims(), 0, transposeResults, contractOp.getContext()); operands[it.index()] = rewriter.create( - contractOp.getLoc(), operands[it.index()], perm); + loc, operands[it.index()], perm); } } // We have taken care to have the dim to be dropped be @@ -429,18 +428,29 @@ mlir::vector::castAwayContractionLeadingOneDim(vector::ContractionOp contractOp, // Extract if its a valid extraction, otherwise use the operand // without extraction. newOperands.push_back( - validExtract ? rewriter.create(contractOp.getLoc(), - operands[it.index()], - splatZero(dropDim)) + validExtract ? rewriter.create( + loc, operands[it.index()], splatZero(dropDim)) : operands[it.index()]); } - auto newContractOp = rewriter.create( - contractOp.getLoc(), newOperands[0], newOperands[1], newOperands[2], + + // Depending on whether this vector.contract is masked, the replacing Op + // should either be a new vector.contract Op or vector.mask Op. + Operation *newOp = rewriter.create( + loc, newOperands[0], newOperands[1], newOperands[2], rewriter.getAffineMapArrayAttr(newIndexingMaps), rewriter.getArrayAttr(newIteratorTypes), contractOp.getKind()); - rewriter.replaceOpWithNewOp( - contractOp, contractOp->getResultTypes()[0], newContractOp); - return success(); + + if (maskingOp) { + auto newMask = rewriter.create(loc, maskingOp.getMask(), + splatZero(dropDim)); + + newOp = mlir::vector::maskOperation(rewriter, newOp, newMask); + } + + return rewriter + .create(loc, contractOp->getResultTypes()[0], + newOp->getResults()[0]) + .getResult(); } namespace { @@ -450,12 +460,14 @@ namespace { /// 1 dimensions. Also performs tranpose of lhs and rhs operands if required /// prior to extract. struct CastAwayContractionLeadingOneDim - : public OpRewritePattern { - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(vector::ContractionOp contractOp, - PatternRewriter &rewriter) const override { - return castAwayContractionLeadingOneDim(contractOp, rewriter); + : public MaskableOpRewritePattern { + using MaskableOpRewritePattern::MaskableOpRewritePattern; + + FailureOr + matchAndRewriteMaskableOp(vector::ContractionOp contractOp, + MaskingOpInterface maskingOp, + PatternRewriter &rewriter) const override { + return castAwayContractionLeadingOneDim(contractOp, maskingOp, rewriter); } }; diff --git a/mlir/test/Dialect/Vector/vector-dropleadunitdim-transforms.mlir b/mlir/test/Dialect/Vector/vector-dropleadunitdim-transforms.mlir index af6e636245b0..4ba51c5953d1 100644 --- a/mlir/test/Dialect/Vector/vector-dropleadunitdim-transforms.mlir +++ b/mlir/test/Dialect/Vector/vector-dropleadunitdim-transforms.mlir @@ -30,6 +30,80 @@ func.func @cast_away_contraction_leading_one_dims(%arg0: vector<1x16x8xf32>, %ar } // ----- +// CHECK: #[[$MAP_0:.+]] = affine_map<(d0, d1, d2) -> (d0, d2)> +// CHECK: #[[$MAP_1:.+]] = affine_map<(d0, d1, d2) -> (d2, d1)> +// CHECK: #[[$MAP_2:.+]] = affine_map<(d0, d1, d2) -> (d0, d1)> + +// CHECK-LABEL: func.func @cast_away_contraction_leading_one_dim_under_const_mask +// CHECK: %[[MASK:.*]] = vector.constant_mask [15, 15, 8] : vector<16x16x8xi1> +// CHECK: %[[R0:.*]] = vector.extract %{{.*}}[0] : vector<16x8xf32> from vector<1x16x8xf32> +// CHECK: %[[R1:.*]] = vector.extract %{{.*}}[0] : vector<8x16xf32> from vector<1x8x16xf32> +// CHECK: %[[R2:.*]] = vector.extract %{{.*}}[0] : vector<16x16xf32> from vector<1x16x16xf32> +// CHECK: %[[CONTRACT:.*]] = vector.mask %[[MASK]] { +// CHECK-SAME: vector.contract {indexing_maps = [#[[$MAP_0]], #[[$MAP_1]], #[[$MAP_2]]], iterator_types = ["parallel", "parallel", "reduction"], kind = #vector.kind} +// CHECK-SAME: %[[R0]], %[[R1]], %[[R2]] : vector<16x8xf32>, vector<8x16xf32> into vector<16x16xf32> +// CHECK-SAME: } : vector<16x16x8xi1> -> vector<16x16xf32> +// CHECK: %[[RES:.*]] = vector.broadcast %[[CONTRACT]] : vector<16x16xf32> to vector<1x16x16xf32> +// CHECK: return %[[RES]] : vector<1x16x16xf32> + +#contraction_accesses0 = [ + affine_map<(l, i, j, k) -> (l, i, k)>, + affine_map<(l, i, j, k) -> (l, k, j)>, + affine_map<(l, i, j, k) -> (l, i, j)> +] +#contraction_trait0 = { + indexing_maps = #contraction_accesses0, + iterator_types = ["parallel", "parallel", "parallel", "reduction"] +} + +func.func @cast_away_contraction_leading_one_dim_under_const_mask(%arg0: vector<1x16x8xf32>, %arg1: vector<1x8x16xf32>, %arg2: vector<1x16x16xf32>) -> vector<1x16x16xf32> { + %mask = vector.constant_mask [1, 15, 15, 8] : vector<1x16x16x8xi1> + %0 = vector.mask %mask { + vector.contract #contraction_trait0 %arg0, %arg1, %arg2 : vector<1x16x8xf32>, vector<1x8x16xf32> into vector<1x16x16xf32> + } : vector<1x16x16x8xi1> -> vector<1x16x16xf32> + return %0 : vector<1x16x16xf32> +} + +// ----- +// CHECK-DAG: #[[$MAP0:.+]] = affine_map<(d0, d1, d2) -> (d0, d2)> +// CHECK-DAG: #[[$MAP1:.+]] = affine_map<(d0, d1, d2) -> (d2, d1)> +// CHECK-DAG: #[[$MAP2:.+]] = affine_map<(d0, d1, d2) -> (d0, d1)> + +// CHECK-LABEL: func.func @cast_away_contraction_leading_one_dim_under_mask +// CHECK: %[[R0:.*]] = vector.extract %{{.*}} : vector<16x8xf32> from vector<1x16x8xf32> +// CHECK: %[[R1:.*]] = vector.extract %{{.*}} : vector<8x16xf32> from vector<1x8x16xf32> +// CHECK: %[[R2:.*]] = vector.extract %{{.*}} : vector<16x16xf32> from vector<1x16x16xf32> +// CHECK: %[[M:.*]] = vector.extract %{{.*}} : vector<16x16x8xi1> from vector<1x16x16x8xi1> +// CHECK: %[[CONTRACT:.*]] = vector.mask %[[M]] { +// CHECK-SAME: vector.contract {indexing_maps = [#[[$MAP0]], #[[$MAP1]], #[[$MAP2]]], iterator_types = ["parallel", "parallel", "reduction"], kind = #vector.kind} +// CHECK-SAME: %[[R0]], %[[R1]], %[[R2]] : vector<16x8xf32>, vector<8x16xf32> into vector<16x16xf32> +// CHECK-SAME: } : vector<16x16x8xi1> -> vector<16x16xf32> +// CHECK-NEXT: %[[RES:.*]] = vector.broadcast %[[CONTRACT]] : vector<16x16xf32> to vector<1x16x16xf32> +// CHECK-NEXT: return %[[RES]] : vector<1x16x16xf32> + +#contraction_accesses0 = [ + affine_map<(l, i, j, k) -> (l, i, k)>, + affine_map<(l, i, j, k) -> (l, k, j)>, + affine_map<(l, i, j, k) -> (l, i, j)> +] +#contraction_trait0 = { + indexing_maps = #contraction_accesses0, + iterator_types = ["parallel", "parallel", "parallel", "reduction"] +} + +func.func @cast_away_contraction_leading_one_dim_under_mask( + %arg0: vector<1x16x8xf32>, + %arg1: vector<1x8x16xf32>, + %arg2: vector<1x16x16xf32>, + %mask: vector<1x16x16x8xi1>) -> vector<1x16x16xf32> { + %0 = vector.mask %mask { + vector.contract #contraction_trait0 %arg0, %arg1, %arg2 : vector<1x16x8xf32>, vector<1x8x16xf32> into vector<1x16x16xf32> + } : vector<1x16x16x8xi1> -> vector<1x16x16xf32> + return %0: vector<1x16x16xf32> +} + +// ----- + // CHECK-DAG: #[[$map0:.*]] = affine_map<(d0, d1) -> (d1)> // CHECK-DAG: #[[$map1:.*]] = affine_map<(d0, d1) -> (d1, d0)> // CHECK-DAG: #[[$map2:.*]] = affine_map<(d0, d1) -> (d0)> @@ -164,36 +238,6 @@ func.func @cast_away_contraction_leading_one_dims_nonleadingunitdim_rank4_acctra return %0: vector<1x1x2x16xf32> } -// ----- - -// CHECK-DAG: #[[MAP0:.*]] = affine_map<(d0, d1, d2, d3) -> (d0, d1, d3)> -// CHECK-DAG: #[[MAP1:.*]] = affine_map<(d0, d1, d2, d3) -> (d0, d3, d2)> -// CHECK-DAG: #[[MAP2:.*]] = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)> - -// CHECK-LABEL: not_insert_cast_for_contraction_under_mask -// CHECK: %[[MASK:.+]] = vector.constant_mask -// CHECK: %[[CASTED_MASK:.+]] = vector.broadcast %[[MASK]] -// CHECK: %[[RET:.+]] = vector.mask %[[CASTED_MASK]] { -// CHECK-SAME: vector.contract {{.*}} : vector<1x16x8xf32>, vector<1x8x16xf32> into vector<1x16x16xf32> } -// CHECK: return %[[RET]] : vector<1x16x16xf32> - -#contraction_accesses0 = [ - affine_map<(l, i, j, k) -> (l, i, k)>, - affine_map<(l, i, j, k) -> (l, k, j)>, - affine_map<(l, i, j, k) -> (l, i, j)> -] -#contraction_trait0 = { - indexing_maps = #contraction_accesses0, - iterator_types = ["parallel", "parallel", "parallel", "reduction"] -} - -func.func @not_insert_cast_for_contraction_under_mask(%arg0: vector<1x16x8xf32>, %arg1: vector<1x8x16xf32>, %arg2: vector<1x16x16xf32>) -> vector<1x16x16xf32> { - %mask = vector.constant_mask [1, 15, 15, 8] : vector<1x16x16x8xi1> - %0 = vector.mask %mask { - vector.contract #contraction_trait0 %arg0, %arg1, %arg2 : vector<1x16x8xf32>, vector<1x8x16xf32> into vector<1x16x16xf32> - } : vector<1x16x16x8xi1> -> vector<1x16x16xf32> - return %0 : vector<1x16x16xf32> -} // ----- // CHECK-LABEL: func @cast_away_extract_strided_slice_leading_one_dims -- GitLab From 99d8c25b3104fc07f46532bd681515c5f3c71133 Mon Sep 17 00:00:00 2001 From: David Green Date: Fri, 22 Mar 2024 09:55:18 +0000 Subject: [PATCH 030/306] [AArch64] Extra tests for v2i8 concat loads. NFC --- llvm/test/CodeGen/AArch64/insert-subvector.ll | 129 +++++++++++++++++- 1 file changed, 127 insertions(+), 2 deletions(-) diff --git a/llvm/test/CodeGen/AArch64/insert-subvector.ll b/llvm/test/CodeGen/AArch64/insert-subvector.ll index d7656e1cd341..95ad9807ed63 100644 --- a/llvm/test/CodeGen/AArch64/insert-subvector.ll +++ b/llvm/test/CodeGen/AArch64/insert-subvector.ll @@ -374,6 +374,131 @@ define <16 x i8> @load_v16i8_8_2(float %tmp, <16 x i8> %b, ptr %a) { ret <16 x i8> %s2 } +define <8 x i8> @load_v8i8_2_1(float %tmp, <8 x i8> %b, ptr %a) { +; CHECK-LABEL: load_v8i8_2_1: +; CHECK: // %bb.0: +; CHECK-NEXT: ld1 { v2.b }[0], [x0] +; CHECK-NEXT: add x8, x0, #1 +; CHECK-NEXT: mov v0.16b, v2.16b +; CHECK-NEXT: ld1 { v0.b }[4], [x8] +; CHECK-NEXT: mov v2.b[1], v0.b[4] +; CHECK-NEXT: fmov d0, d1 +; CHECK-NEXT: mov v0.h[0], v2.h[0] +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0 +; CHECK-NEXT: ret + %l = load <2 x i8>, ptr %a + %s1 = shufflevector <2 x i8> %l, <2 x i8> poison, <8 x i32> + %s2 = shufflevector <8 x i8> %s1, <8 x i8> %b, <8 x i32> + ret <8 x i8> %s2 +} + +define <8 x i8> @load_v8i8_2_15(float %tmp, <8 x i8> %b, ptr %a) { +; CHECK-LABEL: load_v8i8_2_15: +; CHECK: // %bb.0: +; CHECK-NEXT: ld1 { v0.b }[0], [x0] +; CHECK-NEXT: add x8, x0, #1 +; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1 +; CHECK-NEXT: ld1 { v0.b }[4], [x8] +; CHECK-NEXT: adrp x8, .LCPI33_0 +; CHECK-NEXT: mov v0.b[1], v0.b[4] +; CHECK-NEXT: mov v0.d[1], v1.d[0] +; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI33_0] +; CHECK-NEXT: tbl v0.8b, { v0.16b }, v1.8b +; CHECK-NEXT: ret + %l = load <2 x i8>, ptr %a + %s1 = shufflevector <2 x i8> %l, <2 x i8> poison, <8 x i32> + %s2 = shufflevector <8 x i8> %s1, <8 x i8> %b, <8 x i32> + ret <8 x i8> %s2 +} + +define <8 x i8> @load_v8i8_2_2(float %tmp, <8 x i8> %b, ptr %a) { +; CHECK-LABEL: load_v8i8_2_2: +; CHECK: // %bb.0: +; CHECK-NEXT: ld1 { v2.b }[0], [x0] +; CHECK-NEXT: add x8, x0, #1 +; CHECK-NEXT: mov v0.16b, v2.16b +; CHECK-NEXT: ld1 { v0.b }[4], [x8] +; CHECK-NEXT: mov v2.b[1], v0.b[4] +; CHECK-NEXT: fmov d0, d1 +; CHECK-NEXT: mov v0.h[1], v2.h[0] +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0 +; CHECK-NEXT: ret + %l = load <2 x i8>, ptr %a + %s1 = shufflevector <2 x i8> %l, <2 x i8> poison, <8 x i32> + %s2 = shufflevector <8 x i8> %s1, <8 x i8> %b, <8 x i32> + ret <8 x i8> %s2 +} + +define <8 x i8> @load_v8i8_2_3(float %tmp, <8 x i8> %b, ptr %a) { +; CHECK-LABEL: load_v8i8_2_3: +; CHECK: // %bb.0: +; CHECK-NEXT: ld1 { v2.b }[0], [x0] +; CHECK-NEXT: add x8, x0, #1 +; CHECK-NEXT: mov v0.16b, v2.16b +; CHECK-NEXT: ld1 { v0.b }[4], [x8] +; CHECK-NEXT: mov v2.b[1], v0.b[4] +; CHECK-NEXT: fmov d0, d1 +; CHECK-NEXT: mov v0.h[2], v2.h[0] +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0 +; CHECK-NEXT: ret + %l = load <2 x i8>, ptr %a + %s1 = shufflevector <2 x i8> %l, <2 x i8> poison, <8 x i32> + %s2 = shufflevector <8 x i8> %s1, <8 x i8> %b, <8 x i32> + ret <8 x i8> %s2 +} + +define <8 x i8> @load_v8i8_2_4(float %tmp, <8 x i8> %b, ptr %a) { +; CHECK-LABEL: load_v8i8_2_4: +; CHECK: // %bb.0: +; CHECK-NEXT: ld1 { v2.b }[0], [x0] +; CHECK-NEXT: add x8, x0, #1 +; CHECK-NEXT: mov v0.16b, v2.16b +; CHECK-NEXT: ld1 { v0.b }[4], [x8] +; CHECK-NEXT: mov v2.b[1], v0.b[4] +; CHECK-NEXT: fmov d0, d1 +; CHECK-NEXT: mov v0.h[3], v2.h[0] +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0 +; CHECK-NEXT: ret + %l = load <2 x i8>, ptr %a + %s1 = shufflevector <2 x i8> %l, <2 x i8> poison, <8 x i32> + %s2 = shufflevector <8 x i8> %s1, <8 x i8> %b, <8 x i32> + ret <8 x i8> %s2 +} + +define <4 x i8> @load_v4i8_2_1(float %tmp, <4 x i8> %b, ptr %a) { +; CHECK-LABEL: load_v4i8_2_1: +; CHECK: // %bb.0: +; CHECK-NEXT: ld1 { v0.b }[0], [x0] +; CHECK-NEXT: add x8, x0, #1 +; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1 +; CHECK-NEXT: ld1 { v0.b }[4], [x8] +; CHECK-NEXT: uzp1 v0.4h, v0.4h, v0.4h +; CHECK-NEXT: mov v0.s[1], v1.s[1] +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0 +; CHECK-NEXT: ret + %l = load <2 x i8>, ptr %a + %s1 = shufflevector <2 x i8> %l, <2 x i8> poison, <4 x i32> + %s2 = shufflevector <4 x i8> %s1, <4 x i8> %b, <4 x i32> + ret <4 x i8> %s2 +} + +define <4 x i8> @load_v4i8_2_2(float %tmp, <4 x i8> %b, ptr %a) { +; CHECK-LABEL: load_v4i8_2_2: +; CHECK: // %bb.0: +; CHECK-NEXT: ld1 { v0.b }[0], [x0] +; CHECK-NEXT: add x8, x0, #1 +; CHECK-NEXT: ld1 { v0.b }[4], [x8] +; CHECK-NEXT: uzp1 v2.4h, v0.4h, v0.4h +; CHECK-NEXT: fmov d0, d1 +; CHECK-NEXT: mov v0.s[1], v2.s[0] +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0 +; CHECK-NEXT: ret + %l = load <2 x i8>, ptr %a + %s1 = shufflevector <2 x i8> %l, <2 x i8> poison, <4 x i32> + %s2 = shufflevector <4 x i8> %s1, <4 x i8> %b, <4 x i32> + ret <4 x i8> %s2 +} + ; i16 define <8 x i16> @load_v8i16_2_1(float %tmp, <8 x i16> %b, ptr %a) { @@ -400,10 +525,10 @@ define <8 x i16> @load_v8i16_2_15(float %tmp, <8 x i16> %b, ptr %a) { ; CHECK-NEXT: add x9, x0, #2 ; CHECK-NEXT: // kill: def $q1 killed $q1 def $q0_q1 ; CHECK-NEXT: fmov s2, w8 -; CHECK-NEXT: adrp x8, .LCPI33_0 +; CHECK-NEXT: adrp x8, .LCPI40_0 ; CHECK-NEXT: ld1 { v2.h }[2], [x9] ; CHECK-NEXT: xtn v0.4h, v2.4s -; CHECK-NEXT: ldr q2, [x8, :lo12:.LCPI33_0] +; CHECK-NEXT: ldr q2, [x8, :lo12:.LCPI40_0] ; CHECK-NEXT: tbl v0.16b, { v0.16b, v1.16b }, v2.16b ; CHECK-NEXT: ret %l = load <2 x i16>, ptr %a -- GitLab From 465ea0bfa69aa48afef58666b084467a1c96c81b Mon Sep 17 00:00:00 2001 From: Crefeda Rodrigues <65665931+cfRod@users.noreply.github.com> Date: Fri, 22 Mar 2024 10:08:03 +0000 Subject: [PATCH 031/306] [mlir][vector] Propagate scalability in TransferWriteNonPermutationLowering (#85632) Updates `extendVectorRank` so that scalability in patterns that use it (in particular, `TransferWriteNonPermutationLowering`), is correctly propagated. Closed related previous PR https://github.com/llvm/llvm-project/pull/85270 --------- Signed-off-by: Crefeda Rodrigues Co-authored-by: Benjamin Maxwell --- .../Vector/Transforms/LowerVectorTransfer.cpp | 8 ++++++-- .../vector-transfer-permutation-lowering.mlir | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorTransfer.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorTransfer.cpp index 4a5e8fcfb6ed..0693aa596cb2 100644 --- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorTransfer.cpp +++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorTransfer.cpp @@ -41,8 +41,12 @@ static Value extendVectorRank(OpBuilder &builder, Location loc, Value vec, SmallVector newShape(addedRank, 1); newShape.append(originalVecType.getShape().begin(), originalVecType.getShape().end()); - VectorType newVecType = - VectorType::get(newShape, originalVecType.getElementType()); + + SmallVector newScalableDims(addedRank, false); + newScalableDims.append(originalVecType.getScalableDims().begin(), + originalVecType.getScalableDims().end()); + VectorType newVecType = VectorType::get( + newShape, originalVecType.getElementType(), newScalableDims); return builder.create(loc, newVecType, vec); } diff --git a/mlir/test/Dialect/Vector/vector-transfer-permutation-lowering.mlir b/mlir/test/Dialect/Vector/vector-transfer-permutation-lowering.mlir index 13e07f59a72a..31bd19c0be8e 100644 --- a/mlir/test/Dialect/Vector/vector-transfer-permutation-lowering.mlir +++ b/mlir/test/Dialect/Vector/vector-transfer-permutation-lowering.mlir @@ -41,6 +41,24 @@ func.func @permutation_with_mask_scalable(%2: memref, %dim_1: index, %d return %1 : vector<8x[4]x2xf32> } +// CHECK: func.func @permutation_with_mask_transfer_write_scalable( +// CHECK-SAME: %[[ARG_0:.*]]: vector<4x[8]xi16>, +// CHECK-SAME: %[[ARG_1:.*]]: memref<1x4x?x1x1x1x1xi16>, +// CHECK-SAME: %[[MASK:.*]]: vector<4x[8]xi1>) { +// CHECK: %[[C0:.*]] = arith.constant 0 : index +// CHECK: %[[BCAST_1:.*]] = vector.broadcast %[[ARG_0]] : vector<4x[8]xi16> to vector<1x1x1x1x4x[8]xi16> +// CHECK: %[[BCAST_2:.*]] = vector.broadcast %[[MASK]] : vector<4x[8]xi1> to vector<1x1x1x1x4x[8]xi1> +// CHECK: %[[TRANSPOSE_1:.*]] = vector.transpose %[[BCAST_2]], [4, 5, 0, 1, 2, 3] : vector<1x1x1x1x4x[8]xi1> to vector<4x[8]x1x1x1x1xi1> +// CHECK: %[[TRANSPOSE_2:.*]] = vector.transpose %[[BCAST_1]], [4, 5, 0, 1, 2, 3] : vector<1x1x1x1x4x[8]xi16> to vector<4x[8]x1x1x1x1xi16> +// CHECK: vector.transfer_write %[[TRANSPOSE_2]], %[[ARG_1]]{{\[}}%[[C0]], %[[C0]], %[[C0]], %[[C0]], %[[C0]], %[[C0]], %[[C0]]], %[[TRANSPOSE_1]] {in_bounds = [true, true, true, true, true, true]} : vector<4x[8]x1x1x1x1xi16>, memref<1x4x?x1x1x1x1xi16> +// CHECK: return +func.func @permutation_with_mask_transfer_write_scalable(%arg0: vector<4x[8]xi16>, %arg1: memref<1x4x?x1x1x1x1xi16>, %mask: vector<4x[8]xi1>){ + %c0 = arith.constant 0 : index + vector.transfer_write %arg0, %arg1[%c0, %c0, %c0, %c0, %c0, %c0, %c0], %mask {in_bounds = [true, true], permutation_map = affine_map<(d0, d1, d2, d3, d4, d5, d6) -> (d1, d2)> +} : vector<4x[8]xi16>, memref<1x4x?x1x1x1x1xi16> + + return +} module attributes {transform.with_named_sequence} { transform.named_sequence @__transform_main(%module_op: !transform.any_op {transform.readonly}) { %f = transform.structured.match ops{["func.func"]} in %module_op -- GitLab From de7a50fb88faa1dafee33f10149561936214062b Mon Sep 17 00:00:00 2001 From: jeanPerier Date: Fri, 22 Mar 2024 11:13:04 +0100 Subject: [PATCH 032/306] [flang] Fix lowering of host associated cray pointee symbols (#86121) Cray pointee symbols can be host associated from a module or host procedure while the related cray pointer is not explicitly associated. This caused the "not yet implemented: lowering symbol to HLFIR" to fire when lowering a reference to the cray pointee and fetching the cray pointer. This patch: - Ensures cray pointers are always instantiated when instantiating a cray pointee. - Fix internal procedure lowering to deal with cray pointee host association like it does for pointers (the lowering strategy for cray pointee is to create a pointer that is updated with the cray pointer value before being fetched). This should fix the bug reported in https://github.com/llvm/llvm-project/issues/85420. --- flang/include/flang/Lower/ConvertVariable.h | 6 +- flang/include/flang/Semantics/tools.h | 3 + flang/lib/Lower/Bridge.cpp | 5 +- flang/lib/Lower/ConvertExpr.cpp | 10 +- flang/lib/Lower/ConvertExprToHLFIR.cpp | 9 +- flang/lib/Lower/ConvertVariable.cpp | 48 ++++----- flang/lib/Lower/HostAssociations.cpp | 9 +- flang/lib/Lower/PFTBuilder.cpp | 9 ++ flang/lib/Semantics/tools.cpp | 12 +++ flang/test/Lower/HLFIR/cray-pointers.f90 | 114 ++++++++++++++++++-- flang/test/Lower/cray-pointer.f90 | 4 +- 11 files changed, 176 insertions(+), 53 deletions(-) diff --git a/flang/include/flang/Lower/ConvertVariable.h b/flang/include/flang/Lower/ConvertVariable.h index ab30e317d1d9..d70d3268acac 100644 --- a/flang/include/flang/Lower/ConvertVariable.h +++ b/flang/include/flang/Lower/ConvertVariable.h @@ -161,9 +161,9 @@ void genDeclareSymbol(Fortran::lower::AbstractConverter &converter, fir::FortranVariableFlagsEnum::None, bool force = false); -/// For the given Cray pointee symbol return the corresponding -/// Cray pointer symbol. Assert if the pointer symbol cannot be found. -Fortran::semantics::SymbolRef getCrayPointer(Fortran::semantics::SymbolRef sym); +/// Given the Fortran type of a Cray pointee, return the fir.box type used to +/// track the cray pointee as Fortran pointer. +mlir::Type getCrayPointeeBoxType(mlir::Type); } // namespace lower } // namespace Fortran diff --git a/flang/include/flang/Semantics/tools.h b/flang/include/flang/Semantics/tools.h index dc3cd6c894a2..66774b51316c 100644 --- a/flang/include/flang/Semantics/tools.h +++ b/flang/include/flang/Semantics/tools.h @@ -282,6 +282,9 @@ const Symbol *FindExternallyVisibleObject( // specific procedure of the same name, return it instead. const Symbol &BypassGeneric(const Symbol &); +// Given a cray pointee symbol, returns the related cray pointer symbol. +const Symbol &GetCrayPointer(const Symbol &crayPointee); + using SomeExpr = evaluate::Expr; bool ExprHasTypeCategory( diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp index c3cb9ba6a47e..0b54ee818e3c 100644 --- a/flang/lib/Lower/Bridge.cpp +++ b/flang/lib/Lower/Bridge.cpp @@ -3995,11 +3995,12 @@ private: sym->Rank() == 0) { // get the corresponding Cray pointer - auto ptrSym = Fortran::lower::getCrayPointer(*sym); + const Fortran::semantics::Symbol &ptrSym = + Fortran::semantics::GetCrayPointer(*sym); fir::ExtendedValue ptr = getSymbolExtendedValue(ptrSym, nullptr); mlir::Value ptrVal = fir::getBase(ptr); - mlir::Type ptrTy = genType(*ptrSym); + mlir::Type ptrTy = genType(ptrSym); fir::ExtendedValue pte = getSymbolExtendedValue(*sym, nullptr); diff --git a/flang/lib/Lower/ConvertExpr.cpp b/flang/lib/Lower/ConvertExpr.cpp index d157db2cde49..fb7807718ff8 100644 --- a/flang/lib/Lower/ConvertExpr.cpp +++ b/flang/lib/Lower/ConvertExpr.cpp @@ -862,7 +862,8 @@ public: addr); } else if (sym->test(Fortran::semantics::Symbol::Flag::CrayPointee)) { // get the corresponding Cray pointer - auto ptrSym = Fortran::lower::getCrayPointer(sym); + Fortran::semantics::SymbolRef ptrSym{ + Fortran::semantics::GetCrayPointer(sym)}; ExtValue ptr = gen(ptrSym); mlir::Value ptrVal = fir::getBase(ptr); mlir::Type ptrTy = converter.genType(*ptrSym); @@ -1537,8 +1538,8 @@ public: auto baseSym = getFirstSym(aref); if (baseSym.test(Fortran::semantics::Symbol::Flag::CrayPointee)) { // get the corresponding Cray pointer - auto ptrSym = Fortran::lower::getCrayPointer(baseSym); - + Fortran::semantics::SymbolRef ptrSym{ + Fortran::semantics::GetCrayPointer(baseSym)}; fir::ExtendedValue ptr = gen(ptrSym); mlir::Value ptrVal = fir::getBase(ptr); mlir::Type ptrTy = ptrVal.getType(); @@ -6946,7 +6947,8 @@ private: ComponentPath &components) { mlir::Value ptrVal = nullptr; if (x.test(Fortran::semantics::Symbol::Flag::CrayPointee)) { - auto ptrSym = Fortran::lower::getCrayPointer(x); + Fortran::semantics::SymbolRef ptrSym{ + Fortran::semantics::GetCrayPointer(x)}; ExtValue ptr = converter.getSymbolExtendedValue(ptrSym); ptrVal = fir::getBase(ptr); } diff --git a/flang/lib/Lower/ConvertExprToHLFIR.cpp b/flang/lib/Lower/ConvertExprToHLFIR.cpp index c5bfbdf6b8c1..fe5ce4b17b25 100644 --- a/flang/lib/Lower/ConvertExprToHLFIR.cpp +++ b/flang/lib/Lower/ConvertExprToHLFIR.cpp @@ -284,7 +284,7 @@ private: // value of the Cray pointer variable. fir::FirOpBuilder &builder = getBuilder(); fir::FortranVariableOpInterface ptrVar = - gen(Fortran::lower::getCrayPointer(symbolRef)); + gen(Fortran::semantics::GetCrayPointer(symbolRef)); mlir::Value ptrAddr = ptrVar.getBase(); // Reinterpret the reference to a Cray pointer so that @@ -306,9 +306,16 @@ private: } return *varDef; } + llvm::errs() << *symbolRef << "\n"; TODO(getLoc(), "lowering symbol to HLFIR"); } + fir::FortranVariableOpInterface + gen(const Fortran::semantics::Symbol &symbol) { + Fortran::evaluate::SymbolRef symref{symbol}; + return gen(symref); + } + fir::FortranVariableOpInterface gen(const Fortran::evaluate::Component &component) { if (Fortran::semantics::IsAllocatableOrPointer(component.GetLastSymbol())) diff --git a/flang/lib/Lower/ConvertVariable.cpp b/flang/lib/Lower/ConvertVariable.cpp index 94d849862099..e07ae42dc749 100644 --- a/flang/lib/Lower/ConvertVariable.cpp +++ b/flang/lib/Lower/ConvertVariable.cpp @@ -1554,6 +1554,11 @@ fir::FortranVariableFlagsAttr Fortran::lower::translateSymbolAttributes( mlir::MLIRContext *mlirContext, const Fortran::semantics::Symbol &sym, fir::FortranVariableFlagsEnum extraFlags) { fir::FortranVariableFlagsEnum flags = extraFlags; + if (sym.test(Fortran::semantics::Symbol::Flag::CrayPointee)) { + // CrayPointee are represented as pointers. + flags = flags | fir::FortranVariableFlagsEnum::pointer; + return fir::FortranVariableFlagsAttr::get(mlirContext, flags); + } const auto &attrs = sym.attrs(); if (attrs.test(Fortran::semantics::Attr::ALLOCATABLE)) flags = flags | fir::FortranVariableFlagsEnum::allocatable; @@ -1615,8 +1620,6 @@ static void genDeclareSymbol(Fortran::lower::AbstractConverter &converter, (!Fortran::semantics::IsProcedure(sym) || Fortran::semantics::IsPointer(sym)) && !sym.detailsIf()) { - bool isCrayPointee = - sym.test(Fortran::semantics::Symbol::Flag::CrayPointee); fir::FirOpBuilder &builder = converter.getFirOpBuilder(); const mlir::Location loc = genLocation(converter, sym); mlir::Value shapeOrShift; @@ -1636,31 +1639,21 @@ static void genDeclareSymbol(Fortran::lower::AbstractConverter &converter, Fortran::lower::translateSymbolCUDADataAttribute(builder.getContext(), sym); - if (isCrayPointee) { - mlir::Type baseType = - hlfir::getFortranElementOrSequenceType(base.getType()); - if (auto seqType = mlir::dyn_cast(baseType)) { - // The pointer box's sequence type must be with unknown shape. - llvm::SmallVector shape(seqType.getDimension(), - fir::SequenceType::getUnknownExtent()); - baseType = fir::SequenceType::get(shape, seqType.getEleTy()); - } - fir::BoxType ptrBoxType = - fir::BoxType::get(fir::PointerType::get(baseType)); + if (sym.test(Fortran::semantics::Symbol::Flag::CrayPointee)) { + mlir::Type ptrBoxType = + Fortran::lower::getCrayPointeeBoxType(base.getType()); mlir::Value boxAlloc = builder.createTemporary(loc, ptrBoxType); // Declare a local pointer variable. - attributes = fir::FortranVariableFlagsAttr::get( - builder.getContext(), fir::FortranVariableFlagsEnum::pointer); auto newBase = builder.create( loc, boxAlloc, name, /*shape=*/nullptr, lenParams, attributes); - mlir::Value nullAddr = - builder.createNullConstant(loc, ptrBoxType.getEleTy()); + mlir::Value nullAddr = builder.createNullConstant( + loc, llvm::cast(ptrBoxType).getEleTy()); // If the element type is known-length character, then // EmboxOp does not need the length parameters. if (auto charType = mlir::dyn_cast( - fir::unwrapSequenceType(baseType))) + hlfir::getFortranElementType(base.getType()))) if (!charType.hasDynamicLen()) lenParams.clear(); @@ -2346,16 +2339,13 @@ void Fortran::lower::createRuntimeTypeInfoGlobal( defineGlobal(converter, var, globalName, linkage); } -Fortran::semantics::SymbolRef -Fortran::lower::getCrayPointer(Fortran::semantics::SymbolRef sym) { - assert(!sym->GetUltimate().owner().crayPointers().empty() && - "empty Cray pointer/pointee map"); - for (const auto &[pointee, pointer] : - sym->GetUltimate().owner().crayPointers()) { - if (pointee == sym->name()) { - Fortran::semantics::SymbolRef v{pointer.get()}; - return v; - } +mlir::Type Fortran::lower::getCrayPointeeBoxType(mlir::Type fortranType) { + mlir::Type baseType = hlfir::getFortranElementOrSequenceType(fortranType); + if (auto seqType = mlir::dyn_cast(baseType)) { + // The pointer box's sequence type must be with unknown shape. + llvm::SmallVector shape(seqType.getDimension(), + fir::SequenceType::getUnknownExtent()); + baseType = fir::SequenceType::get(shape, seqType.getEleTy()); } - llvm_unreachable("corresponding Cray pointer cannot be found"); + return fir::BoxType::get(fir::PointerType::get(baseType)); } diff --git a/flang/lib/Lower/HostAssociations.cpp b/flang/lib/Lower/HostAssociations.cpp index 414673b00f44..8eb548eb2bd5 100644 --- a/flang/lib/Lower/HostAssociations.cpp +++ b/flang/lib/Lower/HostAssociations.cpp @@ -315,7 +315,11 @@ class CapturedAllocatableAndPointer public: static mlir::Type getType(Fortran::lower::AbstractConverter &converter, const Fortran::semantics::Symbol &sym) { - return fir::ReferenceType::get(converter.genType(sym)); + mlir::Type baseType = converter.genType(sym); + if (sym.GetUltimate().test(Fortran::semantics::Symbol::Flag::CrayPointee)) + return fir::ReferenceType::get( + Fortran::lower::getCrayPointeeBoxType(baseType)); + return fir::ReferenceType::get(baseType); } static void instantiateHostTuple(const InstantiateHostTuple &args, Fortran::lower::AbstractConverter &converter, @@ -507,7 +511,8 @@ walkCaptureCategories(T visitor, Fortran::lower::AbstractConverter &converter, if (Fortran::semantics::IsProcedure(sym)) return CapturedProcedure::visit(visitor, converter, sym, ba); ba.analyze(sym); - if (Fortran::semantics::IsAllocatableOrPointer(sym)) + if (Fortran::semantics::IsAllocatableOrPointer(sym) || + sym.GetUltimate().test(Fortran::semantics::Symbol::Flag::CrayPointee)) return CapturedAllocatableAndPointer::visit(visitor, converter, sym, ba); if (ba.isArray()) return CapturedArrays::visit(visitor, converter, sym, ba); diff --git a/flang/lib/Lower/PFTBuilder.cpp b/flang/lib/Lower/PFTBuilder.cpp index 1dacd5cf64cd..f196b9c5a0cb 100644 --- a/flang/lib/Lower/PFTBuilder.cpp +++ b/flang/lib/Lower/PFTBuilder.cpp @@ -1594,6 +1594,11 @@ private: if (!s->has()) depth = std::max(analyze(s) + 1, depth); } + + // Make sure cray pointer is instantiated even if it is not visible. + if (ultimate.test(Fortran::semantics::Symbol::Flag::CrayPointee)) + depth = std::max( + analyze(Fortran::semantics::GetCrayPointer(ultimate)) + 1, depth); adjustSize(depth + 1); bool global = lower::symbolIsGlobal(sym); layeredVarList[depth].emplace_back(sym, global, depth); @@ -2002,6 +2007,10 @@ struct SymbolVisitor { } } } + // - CrayPointer needs to be available whenever a CrayPointee is used. + if (symbol.GetUltimate().test( + Fortran::semantics::Symbol::Flag::CrayPointee)) + visitSymbol(Fortran::semantics::GetCrayPointer(symbol)); } template diff --git a/flang/lib/Semantics/tools.cpp b/flang/lib/Semantics/tools.cpp index 0484baae93cd..2230047abd72 100644 --- a/flang/lib/Semantics/tools.cpp +++ b/flang/lib/Semantics/tools.cpp @@ -403,6 +403,18 @@ const Symbol &BypassGeneric(const Symbol &symbol) { return symbol; } +const Symbol &GetCrayPointer(const Symbol &crayPointee) { + const Symbol *found{nullptr}; + for (const auto &[pointee, pointer] : + crayPointee.GetUltimate().owner().crayPointers()) { + if (pointee == crayPointee.name()) { + found = &pointer.get(); + break; + } + } + return DEREF(found); +} + bool ExprHasTypeCategory( const SomeExpr &expr, const common::TypeCategory &type) { auto dynamicType{expr.GetType()}; diff --git a/flang/test/Lower/HLFIR/cray-pointers.f90 b/flang/test/Lower/HLFIR/cray-pointers.f90 index d1f1a5647ff1..d969aa5d747a 100644 --- a/flang/test/Lower/HLFIR/cray-pointers.f90 +++ b/flang/test/Lower/HLFIR/cray-pointers.f90 @@ -204,14 +204,14 @@ end subroutine test7 ! CHECK: %[[VAL_3:.*]] = fir.alloca !fir.array<5xi32> {bindc_name = "arr", uniq_name = "_QFtest7Earr"} ! CHECK: %[[VAL_4:.*]] = fir.shape %[[VAL_2]] : (index) -> !fir.shape<1> ! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_3]](%[[VAL_4]]) {uniq_name = "_QFtest7Earr"} : (!fir.ref>, !fir.shape<1>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[VAL_12:.*]] = fir.alloca i64 {bindc_name = "ptr", uniq_name = "_QFtest7Eptr"} +! CHECK: %[[VAL_13:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFtest7Eptr"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_6:.*]] = arith.constant 5 : index ! CHECK: %[[VAL_8:.*]] = fir.shape %[[VAL_6]] : (index) -> !fir.shape<1> ! CHECK: %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_1]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFtest7Epte"} : (!fir.ref>>>) -> (!fir.ref>>>, !fir.ref>>>) ! CHECK: %[[VAL_10:.*]] = fir.zero_bits !fir.ptr> ! CHECK: %[[VAL_11:.*]] = fir.embox %[[VAL_10]](%[[VAL_8]]) : (!fir.ptr>, !fir.shape<1>) -> !fir.box>> ! CHECK: fir.store %[[VAL_11]] to %[[VAL_9]]#0 : !fir.ref>>> -! CHECK: %[[VAL_12:.*]] = fir.alloca i64 {bindc_name = "ptr", uniq_name = "_QFtest7Eptr"} -! CHECK: %[[VAL_13:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFtest7Eptr"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_14:.*]] = fir.convert %[[VAL_13]]#0 : (!fir.ref) -> !fir.ref> ! CHECK: %[[VAL_15:.*]] = fir.load %[[VAL_14]] : !fir.ref> ! CHECK: %[[VAL_16:.*]] = fir.convert %[[VAL_9]]#0 : (!fir.ref>>>) -> !fir.ref> @@ -226,14 +226,14 @@ subroutine test8() end subroutine test8 ! CHECK-LABEL: func.func @_QPtest8( ! CHECK: %[[VAL_1:.*]] = fir.alloca !fir.box>> +! CHECK: %[[VAL_8:.*]] = fir.alloca i64 {bindc_name = "ptr", uniq_name = "_QFtest8Eptr"} +! CHECK: %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_8]] {uniq_name = "_QFtest8Eptr"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_2:.*]] = arith.constant 5 : index ! CHECK: %[[VAL_4:.*]] = fir.shape %[[VAL_2]] : (index) -> !fir.shape<1> ! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_1]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFtest8Epte"} : (!fir.ref>>>) -> (!fir.ref>>>, !fir.ref>>>) ! CHECK: %[[VAL_6:.*]] = fir.zero_bits !fir.ptr> ! CHECK: %[[VAL_7:.*]] = fir.embox %[[VAL_6]](%[[VAL_4]]) : (!fir.ptr>, !fir.shape<1>) -> !fir.box>> ! CHECK: fir.store %[[VAL_7]] to %[[VAL_5]]#0 : !fir.ref>>> -! CHECK: %[[VAL_8:.*]] = fir.alloca i64 {bindc_name = "ptr", uniq_name = "_QFtest8Eptr"} -! CHECK: %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_8]] {uniq_name = "_QFtest8Eptr"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_10:.*]] = fir.convert %[[VAL_9]]#0 : (!fir.ref) -> !fir.ref> ! CHECK: %[[VAL_11:.*]] = fir.load %[[VAL_10]] : !fir.ref> ! CHECK: %[[VAL_12:.*]] = fir.convert %[[VAL_5]]#0 : (!fir.ref>>>) -> !fir.ref> @@ -256,14 +256,14 @@ subroutine test9() end subroutine test9 ! CHECK-LABEL: func.func @_QPtest9( ! CHECK: %[[VAL_1:.*]] = fir.alloca !fir.box>> +! CHECK: %[[VAL_8:.*]] = fir.alloca i64 {bindc_name = "ptr", uniq_name = "_QFtest9Eptr"} +! CHECK: %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_8]] {uniq_name = "_QFtest9Eptr"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_2:.*]] = arith.constant 5 : index ! CHECK: %[[VAL_4:.*]] = fir.shape %[[VAL_2]] : (index) -> !fir.shape<1> ! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_1]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFtest9Epte"} : (!fir.ref>>>) -> (!fir.ref>>>, !fir.ref>>>) ! CHECK: %[[VAL_6:.*]] = fir.zero_bits !fir.ptr> ! CHECK: %[[VAL_7:.*]] = fir.embox %[[VAL_6]](%[[VAL_4]]) : (!fir.ptr>, !fir.shape<1>) -> !fir.box>> ! CHECK: fir.store %[[VAL_7]] to %[[VAL_5]]#0 : !fir.ref>>> -! CHECK: %[[VAL_8:.*]] = fir.alloca i64 {bindc_name = "ptr", uniq_name = "_QFtest9Eptr"} -! CHECK: %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_8]] {uniq_name = "_QFtest9Eptr"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_10:.*]] = fir.convert %[[VAL_9]]#0 : (!fir.ref) -> !fir.ref> ! CHECK: %[[VAL_11:.*]] = fir.load %[[VAL_10]] : !fir.ref> ! CHECK: %[[VAL_12:.*]] = fir.convert %[[VAL_5]]#0 : (!fir.ref>>>) -> !fir.ref> @@ -287,12 +287,12 @@ subroutine test10() end subroutine test10 ! CHECK-LABEL: func.func @_QPtest10( ! CHECK: %[[VAL_1:.*]] = fir.alloca !fir.box> +! CHECK: %[[VAL_6:.*]] = fir.alloca i64 {bindc_name = "ptr", uniq_name = "_QFtest10Eptr"} +! CHECK: %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFtest10Eptr"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFtest10Epte"} : (!fir.ref>>) -> (!fir.ref>>, !fir.ref>>) ! CHECK: %[[VAL_4:.*]] = fir.zero_bits !fir.ptr ! CHECK: %[[VAL_5:.*]] = fir.embox %[[VAL_4]] : (!fir.ptr) -> !fir.box> ! CHECK: fir.store %[[VAL_5]] to %[[VAL_3]]#0 : !fir.ref>> -! CHECK: %[[VAL_6:.*]] = fir.alloca i64 {bindc_name = "ptr", uniq_name = "_QFtest10Eptr"} -! CHECK: %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFtest10Eptr"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_8:.*]] = fir.convert %[[VAL_7]]#0 : (!fir.ref) -> !fir.ref> ! CHECK: %[[VAL_9:.*]] = fir.load %[[VAL_8]] : !fir.ref> ! CHECK: %[[VAL_10:.*]] = fir.convert %[[VAL_3]]#0 : (!fir.ref>>) -> !fir.ref> @@ -315,12 +315,12 @@ subroutine test11() end subroutine test11 ! CHECK-LABEL: func.func @_QPtest11( ! CHECK: %[[VAL_1:.*]] = fir.alloca !fir.box> +! CHECK: %[[VAL_6:.*]] = fir.alloca i64 {bindc_name = "ptr", uniq_name = "_QFtest11Eptr"} +! CHECK: %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFtest11Eptr"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFtest11Epte"} : (!fir.ref>>) -> (!fir.ref>>, !fir.ref>>) ! CHECK: %[[VAL_4:.*]] = fir.zero_bits !fir.ptr ! CHECK: %[[VAL_5:.*]] = fir.embox %[[VAL_4]] : (!fir.ptr) -> !fir.box> ! CHECK: fir.store %[[VAL_5]] to %[[VAL_3]]#0 : !fir.ref>> -! CHECK: %[[VAL_6:.*]] = fir.alloca i64 {bindc_name = "ptr", uniq_name = "_QFtest11Eptr"} -! CHECK: %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFtest11Eptr"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_8:.*]] = fir.convert %[[VAL_7]]#0 : (!fir.ref) -> !fir.ref> ! CHECK: %[[VAL_9:.*]] = fir.load %[[VAL_8]] : !fir.ref> ! CHECK: %[[VAL_10:.*]] = fir.convert %[[VAL_3]]#0 : (!fir.ref>>) -> !fir.ref> @@ -330,3 +330,97 @@ end subroutine test11 ! CHECK: %[[VAL_14:.*]] = fir.box_addr %[[VAL_13]] : (!fir.box>) -> !fir.ptr ! CHECK: %[[VAL_15:.*]] = fir.load %[[VAL_14]] : !fir.ptr ! CHECK: fir.call @_QPsub2(%[[VAL_15]]) fastmath : (i32) -> () + +module test_mod + integer(8) :: cray_pointer + real :: cray_pointee + pointer(cray_pointer, cray_pointee) +end module + +subroutine test_hidden_pointer + ! Only the pointee is accessed, yet the pointer is needed + ! for lowering. + use test_mod, only : cray_pointee + call takes_real(cray_pointee) +end +! CHECK-LABEL: func.func @_QPtest_hidden_pointer() { +! CHECK: %[[VAL_0:.*]] = fir.alloca !fir.box> +! CHECK: %[[VAL_1:.*]] = fir.address_of(@_QMtest_modEcray_pointer) : !fir.ref +! CHECK: %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QMtest_modEcray_pointer"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QMtest_modEcray_pointee"} : (!fir.ref>>) -> (!fir.ref>>, !fir.ref>>) +! CHECK: %[[VAL_4:.*]] = fir.zero_bits !fir.ptr +! CHECK: %[[VAL_5:.*]] = fir.embox %[[VAL_4]] : (!fir.ptr) -> !fir.box> +! CHECK: fir.store %[[VAL_5]] to %[[VAL_3]]#0 : !fir.ref>> +! CHECK: %[[VAL_6:.*]] = fir.convert %[[VAL_2]]#0 : (!fir.ref) -> !fir.ref> +! CHECK: %[[VAL_7:.*]] = fir.load %[[VAL_6]] : !fir.ref> +! CHECK: %[[VAL_8:.*]] = fir.convert %[[VAL_3]]#0 : (!fir.ref>>) -> !fir.ref> +! CHECK: %[[VAL_9:.*]] = fir.convert %[[VAL_7]] : (!fir.ptr) -> !fir.llvm_ptr +! CHECK: %[[VAL_10:.*]] = fir.call @_FortranAPointerAssociateScalar(%[[VAL_8]], %[[VAL_9]]) fastmath : (!fir.ref>, !fir.llvm_ptr) -> none +! CHECK: %[[VAL_11:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref>> +! CHECK: %[[VAL_12:.*]] = fir.box_addr %[[VAL_11]] : (!fir.box>) -> !fir.ptr +! CHECK: %[[VAL_13:.*]] = fir.convert %[[VAL_12]] : (!fir.ptr) -> !fir.ref +! CHECK: fir.call @_QPtakes_real(%[[VAL_13]]) fastmath : (!fir.ref) -> () +! CHECK: return +! CHECK: } + + + +subroutine test_craypointer_capture(n) + integer :: n + character(n) :: cray_pointee + integer(8) :: cray_pointer + pointer(cray_pointer, cray_pointee) + call internal() + contains +subroutine internal() + call takes_character(cray_pointee) +end subroutine +end subroutine +! CHECK-LABEL: func.func @_QPtest_craypointer_capture( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref {fir.bindc_name = "n"}) { +! CHECK: %[[VAL_1:.*]] = fir.alloca !fir.box>> +! CHECK: %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest_craypointer_captureEn"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_3:.*]] = fir.alloca i64 {bindc_name = "cray_pointer", uniq_name = "_QFtest_craypointer_captureEcray_pointer"} +! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFtest_craypointer_captureEcray_pointer"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_5:.*]] = fir.load %[[VAL_2]]#0 : !fir.ref +! CHECK: %[[VAL_6:.*]] = arith.constant 0 : i32 +! CHECK: %[[VAL_7:.*]] = arith.cmpi sgt, %[[VAL_5]], %[[VAL_6]] : i32 +! CHECK: %[[VAL_8:.*]] = arith.select %[[VAL_7]], %[[VAL_5]], %[[VAL_6]] : i32 +! CHECK: %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_1]] typeparams %[[VAL_8]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFtest_craypointer_captureEcray_pointee"} : (!fir.ref>>>, i32) -> (!fir.ref>>>, !fir.ref>>>) +! CHECK: %[[VAL_10:.*]] = fir.zero_bits !fir.ptr> +! CHECK: %[[VAL_11:.*]] = fir.embox %[[VAL_10]] typeparams %[[VAL_8]] : (!fir.ptr>, i32) -> !fir.box>> +! CHECK: fir.store %[[VAL_11]] to %[[VAL_9]]#0 : !fir.ref>>> +! CHECK: %[[VAL_12:.*]] = fir.alloca tuple>>>, !fir.ref> +! CHECK: %[[VAL_13:.*]] = arith.constant 0 : i32 +! CHECK: %[[VAL_14:.*]] = fir.coordinate_of %[[VAL_12]], %[[VAL_13]] : (!fir.ref>>>, !fir.ref>>, i32) -> !fir.llvm_ptr>>>> +! CHECK: fir.store %[[VAL_9]]#1 to %[[VAL_14]] : !fir.llvm_ptr>>>> +! CHECK: %[[VAL_15:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_16:.*]] = fir.coordinate_of %[[VAL_12]], %[[VAL_15]] : (!fir.ref>>>, !fir.ref>>, i32) -> !fir.llvm_ptr> +! CHECK: fir.store %[[VAL_4]]#1 to %[[VAL_16]] : !fir.llvm_ptr> +! CHECK: fir.call @_QFtest_craypointer_capturePinternal(%[[VAL_12]]) fastmath : (!fir.ref>>>, !fir.ref>>) -> () +! CHECK: return +! CHECK: } + +! CHECK-LABEL: func.func private @_QFtest_craypointer_capturePinternal( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref>>>, !fir.ref>> {fir.host_assoc}) +! CHECK: %[[VAL_1:.*]] = arith.constant 0 : i32 +! CHECK: %[[VAL_2:.*]] = fir.coordinate_of %[[VAL_0]], %[[VAL_1]] : (!fir.ref>>>, !fir.ref>>, i32) -> !fir.llvm_ptr>>>> +! CHECK: %[[VAL_3:.*]] = fir.load %[[VAL_2]] : !fir.llvm_ptr>>>> +! CHECK: %[[VAL_4:.*]] = fir.load %[[VAL_3]] : !fir.ref>>> +! CHECK: %[[VAL_5:.*]] = fir.box_elesize %[[VAL_4]] : (!fir.box>>) -> index +! CHECK: %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_3]] typeparams %[[VAL_5]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFtest_craypointer_captureEcray_pointee"} : (!fir.ref>>>, index) -> (!fir.ref>>>, !fir.ref>>>) +! CHECK: %[[VAL_7:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_8:.*]] = fir.coordinate_of %[[VAL_0]], %[[VAL_7]] : (!fir.ref>>>, !fir.ref>>, i32) -> !fir.llvm_ptr> +! CHECK: %[[VAL_9:.*]] = fir.load %[[VAL_8]] : !fir.llvm_ptr> +! CHECK: %[[VAL_10:.*]]:2 = hlfir.declare %[[VAL_9]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFtest_craypointer_captureEcray_pointer"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_11:.*]] = fir.convert %[[VAL_10]]#0 : (!fir.ref) -> !fir.ref> +! CHECK: %[[VAL_12:.*]] = fir.load %[[VAL_11]] : !fir.ref> +! CHECK: %[[VAL_13:.*]] = fir.convert %[[VAL_6]]#0 : (!fir.ref>>>) -> !fir.ref> +! CHECK: %[[VAL_14:.*]] = fir.convert %[[VAL_12]] : (!fir.ptr) -> !fir.llvm_ptr +! CHECK: %[[VAL_15:.*]] = fir.call @_FortranAPointerAssociateScalar(%[[VAL_13]], %[[VAL_14]]) fastmath : (!fir.ref>, !fir.llvm_ptr) -> none +! CHECK: %[[VAL_16:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref>>> +! CHECK: %[[VAL_17:.*]] = fir.box_addr %[[VAL_16]] : (!fir.box>>) -> !fir.ptr> +! CHECK: %[[VAL_18:.*]] = fir.emboxchar %[[VAL_17]], %[[VAL_5]] : (!fir.ptr>, index) -> !fir.boxchar<1> +! CHECK: fir.call @_QPtakes_character(%[[VAL_18]]) fastmath : (!fir.boxchar<1>) -> () +! CHECK: return +! CHECK: } diff --git a/flang/test/Lower/cray-pointer.f90 b/flang/test/Lower/cray-pointer.f90 index 4e9f49daab4e..06910bce35a1 100644 --- a/flang/test/Lower/cray-pointer.f90 +++ b/flang/test/Lower/cray-pointer.f90 @@ -264,8 +264,8 @@ subroutine cray_array() ! CHECK: %[[data:.*]] = fir.alloca !fir.array<5xi32> {{.*}} ! CHECK: %[[c3:.*]] = arith.constant 3 : index ! CHECK: %[[k:.*]] = fir.alloca !fir.array<3xi32> {{.*}} -! CHECK: %[[c31:.*]] = arith.constant 3 : index ! CHECK: %[[ptr:.*]] = fir.alloca i64 {{.*}} +! CHECK: %[[c31:.*]] = arith.constant 3 : index ! CHECK: %[[c2:.*]] = arith.constant 2 : i64 ! CHECK: %[[c1:.*]] = arith.constant 1 : i64 ! CHECK: %[[sub:.*]] = arith.subi %[[c2]], %[[c1]] : i64 @@ -327,8 +327,8 @@ subroutine cray_arraySection() ! CHECK: %[[data:.*]] = fir.alloca !fir.array<5xi32> {{.*}} ! CHECK: %[[c2:.*]] = arith.constant 2 : index ! CHECK: %[[k:.*]] = fir.alloca !fir.array<2xi32> {{.*}} -! CHECK: %[[c3:.*]] = arith.constant 3 : index ! CHECK: %[[ptr:.*]] = fir.alloca i64 {{.*}} +! CHECK: %[[c3:.*]] = arith.constant 3 : index ! CHECK: %[[c1:.*]] = arith.constant 2 : i64 ! CHECK: %[[c0:.*]] = arith.constant 1 : i64 ! CHECK: %[[sub:.*]] = arith.subi %[[c1]], %[[c0]] : i64 -- GitLab From 66f88de80599ec4461b0fdac3d1e396b6e83052d Mon Sep 17 00:00:00 2001 From: Wang Pengcheng Date: Fri, 22 Mar 2024 18:24:23 +0800 Subject: [PATCH 033/306] [RISCV] Support RISC-V Profiles in -march option (#76357) This PR implements the draft https://github.com/riscv-non-isa/riscv-toolchain-conventions/pull/36. Currently, we replace specified profile in `-march` with standard arch string. --- clang/docs/ReleaseNotes.rst | 1 + clang/test/Driver/riscv-profiles.c | 312 +++++++++++++++++++++++++++++ llvm/lib/Support/RISCVISAInfo.cpp | 64 ++++++ 3 files changed, 377 insertions(+) create mode 100644 clang/test/Driver/riscv-profiles.c diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index fd12bb41be47..005cdebc0d8a 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -520,6 +520,7 @@ RISC-V Support ^^^^^^^^^^^^^^ - ``__attribute__((rvv_vector_bits(N)))`` is now supported for RVV vbool*_t types. +- Profile names in ``-march`` option are now supported. CUDA/HIP Language Changes ^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/clang/test/Driver/riscv-profiles.c b/clang/test/Driver/riscv-profiles.c new file mode 100644 index 000000000000..904f0c371f44 --- /dev/null +++ b/clang/test/Driver/riscv-profiles.c @@ -0,0 +1,312 @@ +// RUN: %clang -### -c %s 2>&1 -march=rvi20u32 | FileCheck -check-prefix=RVI20U32 %s +// RVI20U32: "-target-feature" "-a" +// RVI20U32: "-target-feature" "-c" +// RVI20U32: "-target-feature" "-d" +// RVI20U32: "-target-feature" "-f" +// RVI20U32: "-target-feature" "-m" + +// RUN: %clang -### -c %s 2>&1 -march=rvi20u64 | FileCheck -check-prefix=RVI20U64 %s +// RVI20U64: "-target-feature" "-a" +// RVI20U64: "-target-feature" "-c" +// RVI20U64: "-target-feature" "-d" +// RVI20U64: "-target-feature" "-f" +// RVI20U64: "-target-feature" "-m" + +// RUN: %clang -### -c %s 2>&1 -march=rva20u64 | FileCheck -check-prefix=RVA20U64 %s +// RVA20U64: "-target-feature" "+m" +// RVA20U64: "-target-feature" "+a" +// RVA20U64: "-target-feature" "+f" +// RVA20U64: "-target-feature" "+d" +// RVA20U64: "-target-feature" "+c" +// RVA20U64: "-target-feature" "+ziccamoa" +// RVA20U64: "-target-feature" "+ziccif" +// RVA20U64: "-target-feature" "+zicclsm" +// RVA20U64: "-target-feature" "+ziccrse" +// RVA20U64: "-target-feature" "+zicntr" +// RVA20U64: "-target-feature" "+zicsr" +// RVA20U64: "-target-feature" "+za128rs" + +// RUN: %clang -### -c %s 2>&1 -march=rva20s64 | FileCheck -check-prefix=RVA20S64 %s +// RVA20S64: "-target-feature" "+m" +// RVA20S64: "-target-feature" "+a" +// RVA20S64: "-target-feature" "+f" +// RVA20S64: "-target-feature" "+d" +// RVA20S64: "-target-feature" "+c" +// RVA20S64: "-target-feature" "+ziccamoa" +// RVA20S64: "-target-feature" "+ziccif" +// RVA20S64: "-target-feature" "+zicclsm" +// RVA20S64: "-target-feature" "+ziccrse" +// RVA20S64: "-target-feature" "+zicntr" +// RVA20S64: "-target-feature" "+zicsr" +// RVA20S64: "-target-feature" "+zifencei" +// RVA20S64: "-target-feature" "+za128rs" +// RVA20S64: "-target-feature" "+ssccptr" +// RVA20S64: "-target-feature" "+sstvala" +// RVA20S64: "-target-feature" "+sstvecd" +// RVA20S64: "-target-feature" "+svade" +// RVA20S64: "-target-feature" "+svbare" + +// RUN: %clang -### -c %s 2>&1 -march=rva22u64 | FileCheck -check-prefix=RVA22U64 %s +// RVA22U64: "-target-feature" "+m" +// RVA22U64: "-target-feature" "+a" +// RVA22U64: "-target-feature" "+f" +// RVA22U64: "-target-feature" "+d" +// RVA22U64: "-target-feature" "+c" +// RVA22U64: "-target-feature" "+zic64b" +// RVA22U64: "-target-feature" "+zicbom" +// RVA22U64: "-target-feature" "+zicbop" +// RVA22U64: "-target-feature" "+zicboz" +// RVA22U64: "-target-feature" "+ziccamoa" +// RVA22U64: "-target-feature" "+ziccif" +// RVA22U64: "-target-feature" "+zicclsm" +// RVA22U64: "-target-feature" "+ziccrse" +// RVA22U64: "-target-feature" "+zicntr" +// RVA22U64: "-target-feature" "+zicsr" +// RVA22U64: "-target-feature" "+zihintpause" +// RVA22U64: "-target-feature" "+zihpm" +// RVA22U64: "-target-feature" "+za64rs" +// RVA22U64: "-target-feature" "+zfhmin" +// RVA22U64: "-target-feature" "+zba" +// RVA22U64: "-target-feature" "+zbb" +// RVA22U64: "-target-feature" "+zbs" +// RVA22U64: "-target-feature" "+zkt" + +// RUN: %clang -### -c %s 2>&1 -march=rva22s64 | FileCheck -check-prefix=RVA22S64 %s +// RVA22S64: "-target-feature" "+m" +// RVA22S64: "-target-feature" "+a" +// RVA22S64: "-target-feature" "+f" +// RVA22S64: "-target-feature" "+d" +// RVA22S64: "-target-feature" "+c" +// RVA22S64: "-target-feature" "+zic64b" +// RVA22S64: "-target-feature" "+zicbom" +// RVA22S64: "-target-feature" "+zicbop" +// RVA22S64: "-target-feature" "+zicboz" +// RVA22S64: "-target-feature" "+ziccamoa" +// RVA22S64: "-target-feature" "+ziccif" +// RVA22S64: "-target-feature" "+zicclsm" +// RVA22S64: "-target-feature" "+ziccrse" +// RVA22S64: "-target-feature" "+zicntr" +// RVA22S64: "-target-feature" "+zicsr" +// RVA22S64: "-target-feature" "+zifencei" +// RVA22S64: "-target-feature" "+zihintpause" +// RVA22S64: "-target-feature" "+zihpm" +// RVA22S64: "-target-feature" "+za64rs" +// RVA22S64: "-target-feature" "+zfhmin" +// RVA22S64: "-target-feature" "+zba" +// RVA22S64: "-target-feature" "+zbb" +// RVA22S64: "-target-feature" "+zbs" +// RVA22S64: "-target-feature" "+zkt" +// RVA22S64: "-target-feature" "+ssccptr" +// RVA22S64: "-target-feature" "+sscounterenw" +// RVA22S64: "-target-feature" "+sstvala" +// RVA22S64: "-target-feature" "+sstvecd" +// RVA22S64: "-target-feature" "+svade" +// RVA22S64: "-target-feature" "+svbare" +// RVA22S64: "-target-feature" "+svinval" +// RVA22S64: "-target-feature" "+svpbmt" + +// RUN: %clang -### -c %s 2>&1 -march=rva23u64 -menable-experimental-extensions | FileCheck -check-prefix=RVA23U64 %s +// RVA23U64: "-target-feature" "+m" +// RVA23U64: "-target-feature" "+a" +// RVA23U64: "-target-feature" "+f" +// RVA23U64: "-target-feature" "+d" +// RVA23U64: "-target-feature" "+c" +// RVA23U64: "-target-feature" "+v" +// RVA23U64: "-target-feature" "+zic64b" +// RVA23U64: "-target-feature" "+zicbom" +// RVA23U64: "-target-feature" "+zicbop" +// RVA23U64: "-target-feature" "+zicboz" +// RVA23U64: "-target-feature" "+ziccamoa" +// RVA23U64: "-target-feature" "+ziccif" +// RVA23U64: "-target-feature" "+zicclsm" +// RVA23U64: "-target-feature" "+ziccrse" +// RVA23U64: "-target-feature" "+zicntr" +// RVA23U64: "-target-feature" "+zicond" +// RVA23U64: "-target-feature" "+zicsr" +// RVA23U64: "-target-feature" "+zihintntl" +// RVA23U64: "-target-feature" "+zihintpause" +// RVA23U64: "-target-feature" "+zihpm" +// RVA23U64: "-target-feature" "+experimental-zimop" +// RVA23U64: "-target-feature" "+za64rs" +// RVA23U64: "-target-feature" "+zawrs" +// RVA23U64: "-target-feature" "+zfa" +// RVA23U64: "-target-feature" "+zfhmin" +// RVA23U64: "-target-feature" "+zcb" +// RVA23U64: "-target-feature" "+experimental-zcmop" +// RVA23U64: "-target-feature" "+zba" +// RVA23U64: "-target-feature" "+zbb" +// RVA23U64: "-target-feature" "+zbs" +// RVA23U64: "-target-feature" "+zkt" +// RVA23U64: "-target-feature" "+zvbb" +// RVA23U64: "-target-feature" "+zvfhmin" +// RVA23U64: "-target-feature" "+zvkt" + +// RUN: %clang -### -c %s 2>&1 -march=rva23s64 -menable-experimental-extensions | FileCheck -check-prefix=RVA23S64 %s +// RVA23S64: "-target-feature" "+m" +// RVA23S64: "-target-feature" "+a" +// RVA23S64: "-target-feature" "+f" +// RVA23S64: "-target-feature" "+d" +// RVA23S64: "-target-feature" "+c" +// RVA23S64: "-target-feature" "+v" +// RVA23S64: "-target-feature" "+h" +// RVA23S64: "-target-feature" "+zic64b" +// RVA23S64: "-target-feature" "+zicbom" +// RVA23S64: "-target-feature" "+zicbop" +// RVA23S64: "-target-feature" "+zicboz" +// RVA23S64: "-target-feature" "+ziccamoa" +// RVA23S64: "-target-feature" "+ziccif" +// RVA23S64: "-target-feature" "+zicclsm" +// RVA23S64: "-target-feature" "+ziccrse" +// RVA23S64: "-target-feature" "+zicntr" +// RVA23S64: "-target-feature" "+zicond" +// RVA23S64: "-target-feature" "+zicsr" +// RVA23S64: "-target-feature" "+zifencei" +// RVA23S64: "-target-feature" "+zihintntl" +// RVA23S64: "-target-feature" "+zihintpause" +// RVA23S64: "-target-feature" "+zihpm" +// RVA23S64: "-target-feature" "+experimental-zimop" +// RVA23S64: "-target-feature" "+za64rs" +// RVA23S64: "-target-feature" "+zawrs" +// RVA23S64: "-target-feature" "+zfa" +// RVA23S64: "-target-feature" "+zfhmin" +// RVA23S64: "-target-feature" "+zcb" +// RVA23S64: "-target-feature" "+experimental-zcmop" +// RVA23S64: "-target-feature" "+zba" +// RVA23S64: "-target-feature" "+zbb" +// RVA23S64: "-target-feature" "+zbs" +// RVA23S64: "-target-feature" "+zkt" +// RVA23S64: "-target-feature" "+zvbb" +// RVA23S64: "-target-feature" "+zvfhmin" +// RVA23S64: "-target-feature" "+zvkt" +// RVA23S64: "-target-feature" "+shcounterenw" +// RVA23S64: "-target-feature" "+shgatpa" +// RVA23S64: "-target-feature" "+shtvala" +// RVA23S64: "-target-feature" "+shvsatpa" +// RVA23S64: "-target-feature" "+shvstvala" +// RVA23S64: "-target-feature" "+shvstvecd" +// RVA23S64: "-target-feature" "+ssccptr" +// RVA23S64: "-target-feature" "+sscofpmf" +// RVA23S64: "-target-feature" "+sscounterenw" +// RVA23S64: "-target-feature" "+experimental-ssnpm" +// RVA23S64: "-target-feature" "+ssstateen" +// RVA23S64: "-target-feature" "+sstc" +// RVA23S64: "-target-feature" "+sstvala" +// RVA23S64: "-target-feature" "+sstvecd" +// RVA23S64: "-target-feature" "+ssu64xl" +// RVA23S64: "-target-feature" "+svade" +// RVA23S64: "-target-feature" "+svbare" +// RVA23S64: "-target-feature" "+svinval" +// RVA23S64: "-target-feature" "+svnapot" +// RVA23S64: "-target-feature" "+svpbmt" + +// RUN: %clang -### -c %s 2>&1 -march=rvb23u64 -menable-experimental-extensions | FileCheck -check-prefix=RVB23U64 %s +// RVB23U64: "-target-feature" "+m" +// RVB23U64: "-target-feature" "+a" +// RVB23U64: "-target-feature" "+f" +// RVB23U64: "-target-feature" "+d" +// RVB23U64: "-target-feature" "+c" +// RVB23U64: "-target-feature" "+zic64b" +// RVB23U64: "-target-feature" "+zicbom" +// RVB23U64: "-target-feature" "+zicbop" +// RVB23U64: "-target-feature" "+zicboz" +// RVB23U64: "-target-feature" "+ziccamoa" +// RVB23U64: "-target-feature" "+ziccif" +// RVB23U64: "-target-feature" "+zicclsm" +// RVB23U64: "-target-feature" "+ziccrse" +// RVB23U64: "-target-feature" "+zicntr" +// RVB23U64: "-target-feature" "+zicond" +// RVB23U64: "-target-feature" "+zicsr" +// RVB23U64: "-target-feature" "+zihintntl" +// RVB23U64: "-target-feature" "+zihintpause" +// RVB23U64: "-target-feature" "+zihpm" +// RVB23U64: "-target-feature" "+experimental-zimop" +// RVB23U64: "-target-feature" "+za64rs" +// RVB23U64: "-target-feature" "+zawrs" +// RVB23U64: "-target-feature" "+zfa" +// RVB23U64: "-target-feature" "+zcb" +// RVB23U64: "-target-feature" "+experimental-zcmop" +// RVB23U64: "-target-feature" "+zba" +// RVB23U64: "-target-feature" "+zbb" +// RVB23U64: "-target-feature" "+zbs" +// RVB23U64: "-target-feature" "+zkt" + +// RUN: %clang -### -c %s 2>&1 -march=rvb23s64 -menable-experimental-extensions | FileCheck -check-prefix=RVB23S64 %s +// RVB23S64: "-target-feature" "+m" +// RVB23S64: "-target-feature" "+a" +// RVB23S64: "-target-feature" "+f" +// RVB23S64: "-target-feature" "+d" +// RVB23S64: "-target-feature" "+c" +// RVB23S64: "-target-feature" "+zic64b" +// RVB23S64: "-target-feature" "+zicbom" +// RVB23S64: "-target-feature" "+zicbop" +// RVB23S64: "-target-feature" "+zicboz" +// RVB23S64: "-target-feature" "+ziccamoa" +// RVB23S64: "-target-feature" "+ziccif" +// RVB23S64: "-target-feature" "+zicclsm" +// RVB23S64: "-target-feature" "+ziccrse" +// RVB23S64: "-target-feature" "+zicntr" +// RVB23S64: "-target-feature" "+zicond" +// RVB23S64: "-target-feature" "+zicsr" +// RVB23S64: "-target-feature" "+zifencei" +// RVB23S64: "-target-feature" "+zihintntl" +// RVB23S64: "-target-feature" "+zihintpause" +// RVB23S64: "-target-feature" "+zihpm" +// RVB23S64: "-target-feature" "+experimental-zimop" +// RVB23S64: "-target-feature" "+za64rs" +// RVB23S64: "-target-feature" "+zawrs" +// RVB23S64: "-target-feature" "+zfa" +// RVB23S64: "-target-feature" "+zcb" +// RVB23S64: "-target-feature" "+experimental-zcmop" +// RVB23S64: "-target-feature" "+zba" +// RVB23S64: "-target-feature" "+zbb" +// RVB23S64: "-target-feature" "+zbs" +// RVB23S64: "-target-feature" "+zkt" +// RVB23S64: "-target-feature" "+ssccptr" +// RVB23S64: "-target-feature" "+sscofpmf" +// RVB23S64: "-target-feature" "+sscounterenw" +// RVB23S64: "-target-feature" "+sstc" +// RVB23S64: "-target-feature" "+sstvala" +// RVB23S64: "-target-feature" "+sstvecd" +// RVB23S64: "-target-feature" "+ssu64xl" +// RVB23S64: "-target-feature" "+svade" +// RVB23S64: "-target-feature" "+svbare" +// RVB23S64: "-target-feature" "+svinval" +// RVB23S64: "-target-feature" "+svnapot" +// RVB23S64: "-target-feature" "+svpbmt" + +// RUN: %clang -### -c %s 2>&1 -march=rvm23u32 -menable-experimental-extensions | FileCheck -check-prefix=RVM23U32 %s +// RVM23U32: "-target-feature" "+m" +// RVM23U32: "-target-feature" "+zicbop" +// RVM23U32: "-target-feature" "+zicond" +// RVM23U32: "-target-feature" "+zicsr" +// RVM23U32: "-target-feature" "+zihintntl" +// RVM23U32: "-target-feature" "+zihintpause" +// RVM23U32: "-target-feature" "+experimental-zimop" +// RVM23U32: "-target-feature" "+zce" +// RVM23U32: "-target-feature" "+experimental-zcmop" +// RVM23U32: "-target-feature" "+zba" +// RVM23U32: "-target-feature" "+zbb" +// RVM23U32: "-target-feature" "+zbs" + +// RUN: %clang -### -c %s 2>&1 -march=rva22u64_zfa | FileCheck -check-prefix=PROFILE-WITH-ADDITIONAL %s +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+m" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+a" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+f" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+d" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+c" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zicbom" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zicbop" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zicboz" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zihintpause" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zfa" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zfhmin" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zba" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zbb" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zbs" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zkt" + +// RUN: not %clang -### -c %s 2>&1 -march=rva19u64_zfa | FileCheck -check-prefix=INVALID-PROFILE %s +// INVALID-PROFILE: error: invalid arch name 'rva19u64_zfa', unsupported profile + +// RUN: not %clang -### -c %s 2>&1 -march=rva22u64zfa | FileCheck -check-prefix=INVALID-ADDITIONAL %s +// INVALID-ADDITIONAL: error: invalid arch name 'rva22u64zfa', additional extensions must be after separator '_' diff --git a/llvm/lib/Support/RISCVISAInfo.cpp b/llvm/lib/Support/RISCVISAInfo.cpp index 39235ace4724..67e6e5b962b1 100644 --- a/llvm/lib/Support/RISCVISAInfo.cpp +++ b/llvm/lib/Support/RISCVISAInfo.cpp @@ -36,6 +36,11 @@ struct RISCVSupportedExtension { } }; +struct RISCVProfile { + StringLiteral Name; + StringLiteral MArch; +}; + } // end anonymous namespace static constexpr StringLiteral AllStdExts = "mafdqlcbkjtpvnh"; @@ -244,6 +249,42 @@ static const RISCVSupportedExtension SupportedExperimentalExtensions[] = { }; // clang-format on +static constexpr RISCVProfile SupportedProfiles[] = { + {"rvi20u32", "rv32i"}, + {"rvi20u64", "rv64i"}, + {"rva20u64", "rv64imafdc_ziccamoa_ziccif_zicclsm_ziccrse_zicntr_za128rs"}, + {"rva20s64", "rv64imafdc_ziccamoa_ziccif_zicclsm_ziccrse_zicntr_zifencei_" + "za128rs_ssccptr_sstvala_sstvecd_svade_svbare"}, + {"rva22u64", + "rv64imafdc_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_zicclsm_ziccrse_" + "zicntr_zihintpause_zihpm_za64rs_zfhmin_zba_zbb_zbs_zkt"}, + {"rva22s64", + "rv64imafdc_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_zicclsm_ziccrse_" + "zicntr_zifencei_zihintpause_zihpm_za64rs_zfhmin_zba_zbb_zbs_zkt_ssccptr_" + "sscounterenw_sstvala_sstvecd_svade_svbare_svinval_svpbmt"}, + {"rva23u64", + "rv64imafdcv_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_zicclsm_ziccrse_" + "zicntr_zicond_zihintntl_zihintpause_zihpm_zimop0p1_za64rs_zawrs_zfa_" + "zfhmin_zcb_zcmop0p2_zba_zbb_zbs_zkt_zvbb_zvfhmin_zvkt"}, + {"rva23s64", + "rv64imafdcvh_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_zicclsm_ziccrse_" + "zicntr_zicond_zifencei_zihintntl_zihintpause_zihpm_zimop0p1_za64rs_zawrs_" + "zfa_zfhmin_zcb_zcmop0p2_zba_zbb_zbs_zkt_zvbb_zvfhmin_zvkt_shcounterenw_" + "shgatpa_shtvala_shvsatpa_shvstvala_shvstvecd_ssccptr_sscofpmf_" + "sscounterenw_ssnpm0p8_ssstateen_sstc_sstvala_sstvecd_ssu64xl_svade_" + "svbare_svinval_svnapot_svpbmt"}, + {"rvb23u64", "rv64imafdc_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_" + "zicclsm_ziccrse_zicntr_zicond_zihintntl_zihintpause_zihpm_" + "zimop0p1_za64rs_zawrs_zfa_zcb_zcmop0p2_zba_zbb_zbs_zkt"}, + {"rvb23s64", + "rv64imafdc_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_zicclsm_ziccrse_" + "zicntr_zicond_zifencei_zihintntl_zihintpause_zihpm_zimop0p1_za64rs_zawrs_" + "zfa_zcb_zcmop0p2_zba_zbb_zbs_zkt_ssccptr_sscofpmf_sscounterenw_sstc_" + "sstvala_sstvecd_ssu64xl_svade_svbare_svinval_svnapot_svpbmt"}, + {"rvm23u32", "rv32im_zicbop_zicond_zicsr_zihintntl_zihintpause_zimop0p1_" + "zca_zcb_zce_zcmop0p2_zcmp_zcmt_zba_zbb_zbs"}, +}; + static void verifyTables() { #ifndef NDEBUG static std::atomic TableChecked(false); @@ -857,6 +898,29 @@ RISCVISAInfo::parseArchString(StringRef Arch, bool EnableExperimentalExtension, "string must be lowercase"); } + if (Arch.starts_with("rvi") || Arch.starts_with("rva") || + Arch.starts_with("rvb") || Arch.starts_with("rvm")) { + const auto *FoundProfile = + llvm::find_if(SupportedProfiles, [Arch](const RISCVProfile &Profile) { + return Arch.starts_with(Profile.Name); + }); + + if (FoundProfile == std::end(SupportedProfiles)) + return createStringError(errc::invalid_argument, "unsupported profile"); + + std::string NewArch = FoundProfile->MArch.str(); + StringRef ArchWithoutProfile = Arch.substr(FoundProfile->Name.size()); + if (!ArchWithoutProfile.empty()) { + if (!ArchWithoutProfile.starts_with("_")) + return createStringError( + errc::invalid_argument, + "additional extensions must be after separator '_'"); + NewArch += ArchWithoutProfile.str(); + } + return parseArchString(NewArch, EnableExperimentalExtension, + ExperimentalExtensionVersionCheck, IgnoreUnknown); + } + bool HasRV64 = Arch.starts_with("rv64"); // ISA string must begin with rv32 or rv64. if (!(Arch.starts_with("rv32") || HasRV64) || (Arch.size() < 5)) { -- GitLab From cb4453dc69d75064c9a82e9a6a9bf0d0ded4b204 Mon Sep 17 00:00:00 2001 From: XChy Date: Fri, 22 Mar 2024 18:35:20 +0800 Subject: [PATCH 034/306] [SelectionDAG] Prevent combination on inconsistent type in `combineCarryDiamond` (#84888) Fixes #84831 When matching carry pattern with `getAsCarry`, it may produce different type of carryout. This patch checks such case and does early exit. I'm new to DAG, any suggestion is appreciated. --- llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 7 +++++- llvm/test/CodeGen/X86/addcarry.ll | 23 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index 7009f375df11..db81f9199170 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -3473,6 +3473,11 @@ static SDValue combineCarryDiamond(SelectionDAG &DAG, const TargetLowering &TLI, return SDValue(); if (Opcode != ISD::UADDO && Opcode != ISD::USUBO) return SDValue(); + // Guarantee identical type of CarryOut + EVT CarryOutType = N->getValueType(0); + if (CarryOutType != Carry0.getValue(1).getValueType() || + CarryOutType != Carry1.getValue(1).getValueType()) + return SDValue(); // Canonicalize the add/sub of A and B (the top node in the above ASCII art) // as Carry0 and the add/sub of the carry in as Carry1 (the middle node). @@ -3520,7 +3525,7 @@ static SDValue combineCarryDiamond(SelectionDAG &DAG, const TargetLowering &TLI, // TODO: match other operations that can merge flags (ADD, etc) DAG.ReplaceAllUsesOfValueWith(Carry1.getValue(0), Merged.getValue(0)); if (N->getOpcode() == ISD::AND) - return DAG.getConstant(0, DL, MVT::i1); + return DAG.getConstant(0, DL, CarryOutType); return Merged.getValue(1); } diff --git a/llvm/test/CodeGen/X86/addcarry.ll b/llvm/test/CodeGen/X86/addcarry.ll index 3fc4ed99fad0..f8d32fc2d292 100644 --- a/llvm/test/CodeGen/X86/addcarry.ll +++ b/llvm/test/CodeGen/X86/addcarry.ll @@ -1490,3 +1490,26 @@ define { i64, i64 } @addcarry_commutative_2(i64 %x0, i64 %x1, i64 %y0, i64 %y1) %r1 = insertvalue { i64, i64 } %r0, i64 %b1s, 1 ret { i64, i64 } %r1 } + +define i1 @pr84831(i64 %arg) { +; CHECK-LABEL: pr84831: +; CHECK: # %bb.0: +; CHECK-NEXT: testq %rdi, %rdi +; CHECK-NEXT: setne %al +; CHECK-NEXT: xorl %ecx, %ecx +; CHECK-NEXT: addb $-1, %al +; CHECK-NEXT: adcq $1, %rcx +; CHECK-NEXT: setb %al +; CHECK-NEXT: retq + %a = icmp ult i64 0, %arg + %add1 = add i64 0, 1 + %carryout1 = icmp ult i64 %add1, 0 + %b = zext i1 %a to i64 + %add2 = add i64 %add1, %b + %carryout2 = icmp ult i64 %add2, %add1 + %zc1 = zext i1 %carryout1 to i63 + %zc2 = zext i1 %carryout2 to i63 + %or = or i63 %zc1, %zc2 + %trunc = trunc i63 %or to i1 + ret i1 %trunc +} -- GitLab From 6e755c51a916dc521ffe89738bcab47a5442ad06 Mon Sep 17 00:00:00 2001 From: Wang Pengcheng Date: Fri, 22 Mar 2024 18:48:35 +0800 Subject: [PATCH 035/306] Revert "[RISCV] Support RISC-V Profiles in -march option (#76357)" This reverts commit 66f88de80599ec4461b0fdac3d1e396b6e83052d as there are some failures. --- clang/docs/ReleaseNotes.rst | 1 - clang/test/Driver/riscv-profiles.c | 312 ----------------------------- llvm/lib/Support/RISCVISAInfo.cpp | 64 ------ 3 files changed, 377 deletions(-) delete mode 100644 clang/test/Driver/riscv-profiles.c diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index 005cdebc0d8a..fd12bb41be47 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -520,7 +520,6 @@ RISC-V Support ^^^^^^^^^^^^^^ - ``__attribute__((rvv_vector_bits(N)))`` is now supported for RVV vbool*_t types. -- Profile names in ``-march`` option are now supported. CUDA/HIP Language Changes ^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/clang/test/Driver/riscv-profiles.c b/clang/test/Driver/riscv-profiles.c deleted file mode 100644 index 904f0c371f44..000000000000 --- a/clang/test/Driver/riscv-profiles.c +++ /dev/null @@ -1,312 +0,0 @@ -// RUN: %clang -### -c %s 2>&1 -march=rvi20u32 | FileCheck -check-prefix=RVI20U32 %s -// RVI20U32: "-target-feature" "-a" -// RVI20U32: "-target-feature" "-c" -// RVI20U32: "-target-feature" "-d" -// RVI20U32: "-target-feature" "-f" -// RVI20U32: "-target-feature" "-m" - -// RUN: %clang -### -c %s 2>&1 -march=rvi20u64 | FileCheck -check-prefix=RVI20U64 %s -// RVI20U64: "-target-feature" "-a" -// RVI20U64: "-target-feature" "-c" -// RVI20U64: "-target-feature" "-d" -// RVI20U64: "-target-feature" "-f" -// RVI20U64: "-target-feature" "-m" - -// RUN: %clang -### -c %s 2>&1 -march=rva20u64 | FileCheck -check-prefix=RVA20U64 %s -// RVA20U64: "-target-feature" "+m" -// RVA20U64: "-target-feature" "+a" -// RVA20U64: "-target-feature" "+f" -// RVA20U64: "-target-feature" "+d" -// RVA20U64: "-target-feature" "+c" -// RVA20U64: "-target-feature" "+ziccamoa" -// RVA20U64: "-target-feature" "+ziccif" -// RVA20U64: "-target-feature" "+zicclsm" -// RVA20U64: "-target-feature" "+ziccrse" -// RVA20U64: "-target-feature" "+zicntr" -// RVA20U64: "-target-feature" "+zicsr" -// RVA20U64: "-target-feature" "+za128rs" - -// RUN: %clang -### -c %s 2>&1 -march=rva20s64 | FileCheck -check-prefix=RVA20S64 %s -// RVA20S64: "-target-feature" "+m" -// RVA20S64: "-target-feature" "+a" -// RVA20S64: "-target-feature" "+f" -// RVA20S64: "-target-feature" "+d" -// RVA20S64: "-target-feature" "+c" -// RVA20S64: "-target-feature" "+ziccamoa" -// RVA20S64: "-target-feature" "+ziccif" -// RVA20S64: "-target-feature" "+zicclsm" -// RVA20S64: "-target-feature" "+ziccrse" -// RVA20S64: "-target-feature" "+zicntr" -// RVA20S64: "-target-feature" "+zicsr" -// RVA20S64: "-target-feature" "+zifencei" -// RVA20S64: "-target-feature" "+za128rs" -// RVA20S64: "-target-feature" "+ssccptr" -// RVA20S64: "-target-feature" "+sstvala" -// RVA20S64: "-target-feature" "+sstvecd" -// RVA20S64: "-target-feature" "+svade" -// RVA20S64: "-target-feature" "+svbare" - -// RUN: %clang -### -c %s 2>&1 -march=rva22u64 | FileCheck -check-prefix=RVA22U64 %s -// RVA22U64: "-target-feature" "+m" -// RVA22U64: "-target-feature" "+a" -// RVA22U64: "-target-feature" "+f" -// RVA22U64: "-target-feature" "+d" -// RVA22U64: "-target-feature" "+c" -// RVA22U64: "-target-feature" "+zic64b" -// RVA22U64: "-target-feature" "+zicbom" -// RVA22U64: "-target-feature" "+zicbop" -// RVA22U64: "-target-feature" "+zicboz" -// RVA22U64: "-target-feature" "+ziccamoa" -// RVA22U64: "-target-feature" "+ziccif" -// RVA22U64: "-target-feature" "+zicclsm" -// RVA22U64: "-target-feature" "+ziccrse" -// RVA22U64: "-target-feature" "+zicntr" -// RVA22U64: "-target-feature" "+zicsr" -// RVA22U64: "-target-feature" "+zihintpause" -// RVA22U64: "-target-feature" "+zihpm" -// RVA22U64: "-target-feature" "+za64rs" -// RVA22U64: "-target-feature" "+zfhmin" -// RVA22U64: "-target-feature" "+zba" -// RVA22U64: "-target-feature" "+zbb" -// RVA22U64: "-target-feature" "+zbs" -// RVA22U64: "-target-feature" "+zkt" - -// RUN: %clang -### -c %s 2>&1 -march=rva22s64 | FileCheck -check-prefix=RVA22S64 %s -// RVA22S64: "-target-feature" "+m" -// RVA22S64: "-target-feature" "+a" -// RVA22S64: "-target-feature" "+f" -// RVA22S64: "-target-feature" "+d" -// RVA22S64: "-target-feature" "+c" -// RVA22S64: "-target-feature" "+zic64b" -// RVA22S64: "-target-feature" "+zicbom" -// RVA22S64: "-target-feature" "+zicbop" -// RVA22S64: "-target-feature" "+zicboz" -// RVA22S64: "-target-feature" "+ziccamoa" -// RVA22S64: "-target-feature" "+ziccif" -// RVA22S64: "-target-feature" "+zicclsm" -// RVA22S64: "-target-feature" "+ziccrse" -// RVA22S64: "-target-feature" "+zicntr" -// RVA22S64: "-target-feature" "+zicsr" -// RVA22S64: "-target-feature" "+zifencei" -// RVA22S64: "-target-feature" "+zihintpause" -// RVA22S64: "-target-feature" "+zihpm" -// RVA22S64: "-target-feature" "+za64rs" -// RVA22S64: "-target-feature" "+zfhmin" -// RVA22S64: "-target-feature" "+zba" -// RVA22S64: "-target-feature" "+zbb" -// RVA22S64: "-target-feature" "+zbs" -// RVA22S64: "-target-feature" "+zkt" -// RVA22S64: "-target-feature" "+ssccptr" -// RVA22S64: "-target-feature" "+sscounterenw" -// RVA22S64: "-target-feature" "+sstvala" -// RVA22S64: "-target-feature" "+sstvecd" -// RVA22S64: "-target-feature" "+svade" -// RVA22S64: "-target-feature" "+svbare" -// RVA22S64: "-target-feature" "+svinval" -// RVA22S64: "-target-feature" "+svpbmt" - -// RUN: %clang -### -c %s 2>&1 -march=rva23u64 -menable-experimental-extensions | FileCheck -check-prefix=RVA23U64 %s -// RVA23U64: "-target-feature" "+m" -// RVA23U64: "-target-feature" "+a" -// RVA23U64: "-target-feature" "+f" -// RVA23U64: "-target-feature" "+d" -// RVA23U64: "-target-feature" "+c" -// RVA23U64: "-target-feature" "+v" -// RVA23U64: "-target-feature" "+zic64b" -// RVA23U64: "-target-feature" "+zicbom" -// RVA23U64: "-target-feature" "+zicbop" -// RVA23U64: "-target-feature" "+zicboz" -// RVA23U64: "-target-feature" "+ziccamoa" -// RVA23U64: "-target-feature" "+ziccif" -// RVA23U64: "-target-feature" "+zicclsm" -// RVA23U64: "-target-feature" "+ziccrse" -// RVA23U64: "-target-feature" "+zicntr" -// RVA23U64: "-target-feature" "+zicond" -// RVA23U64: "-target-feature" "+zicsr" -// RVA23U64: "-target-feature" "+zihintntl" -// RVA23U64: "-target-feature" "+zihintpause" -// RVA23U64: "-target-feature" "+zihpm" -// RVA23U64: "-target-feature" "+experimental-zimop" -// RVA23U64: "-target-feature" "+za64rs" -// RVA23U64: "-target-feature" "+zawrs" -// RVA23U64: "-target-feature" "+zfa" -// RVA23U64: "-target-feature" "+zfhmin" -// RVA23U64: "-target-feature" "+zcb" -// RVA23U64: "-target-feature" "+experimental-zcmop" -// RVA23U64: "-target-feature" "+zba" -// RVA23U64: "-target-feature" "+zbb" -// RVA23U64: "-target-feature" "+zbs" -// RVA23U64: "-target-feature" "+zkt" -// RVA23U64: "-target-feature" "+zvbb" -// RVA23U64: "-target-feature" "+zvfhmin" -// RVA23U64: "-target-feature" "+zvkt" - -// RUN: %clang -### -c %s 2>&1 -march=rva23s64 -menable-experimental-extensions | FileCheck -check-prefix=RVA23S64 %s -// RVA23S64: "-target-feature" "+m" -// RVA23S64: "-target-feature" "+a" -// RVA23S64: "-target-feature" "+f" -// RVA23S64: "-target-feature" "+d" -// RVA23S64: "-target-feature" "+c" -// RVA23S64: "-target-feature" "+v" -// RVA23S64: "-target-feature" "+h" -// RVA23S64: "-target-feature" "+zic64b" -// RVA23S64: "-target-feature" "+zicbom" -// RVA23S64: "-target-feature" "+zicbop" -// RVA23S64: "-target-feature" "+zicboz" -// RVA23S64: "-target-feature" "+ziccamoa" -// RVA23S64: "-target-feature" "+ziccif" -// RVA23S64: "-target-feature" "+zicclsm" -// RVA23S64: "-target-feature" "+ziccrse" -// RVA23S64: "-target-feature" "+zicntr" -// RVA23S64: "-target-feature" "+zicond" -// RVA23S64: "-target-feature" "+zicsr" -// RVA23S64: "-target-feature" "+zifencei" -// RVA23S64: "-target-feature" "+zihintntl" -// RVA23S64: "-target-feature" "+zihintpause" -// RVA23S64: "-target-feature" "+zihpm" -// RVA23S64: "-target-feature" "+experimental-zimop" -// RVA23S64: "-target-feature" "+za64rs" -// RVA23S64: "-target-feature" "+zawrs" -// RVA23S64: "-target-feature" "+zfa" -// RVA23S64: "-target-feature" "+zfhmin" -// RVA23S64: "-target-feature" "+zcb" -// RVA23S64: "-target-feature" "+experimental-zcmop" -// RVA23S64: "-target-feature" "+zba" -// RVA23S64: "-target-feature" "+zbb" -// RVA23S64: "-target-feature" "+zbs" -// RVA23S64: "-target-feature" "+zkt" -// RVA23S64: "-target-feature" "+zvbb" -// RVA23S64: "-target-feature" "+zvfhmin" -// RVA23S64: "-target-feature" "+zvkt" -// RVA23S64: "-target-feature" "+shcounterenw" -// RVA23S64: "-target-feature" "+shgatpa" -// RVA23S64: "-target-feature" "+shtvala" -// RVA23S64: "-target-feature" "+shvsatpa" -// RVA23S64: "-target-feature" "+shvstvala" -// RVA23S64: "-target-feature" "+shvstvecd" -// RVA23S64: "-target-feature" "+ssccptr" -// RVA23S64: "-target-feature" "+sscofpmf" -// RVA23S64: "-target-feature" "+sscounterenw" -// RVA23S64: "-target-feature" "+experimental-ssnpm" -// RVA23S64: "-target-feature" "+ssstateen" -// RVA23S64: "-target-feature" "+sstc" -// RVA23S64: "-target-feature" "+sstvala" -// RVA23S64: "-target-feature" "+sstvecd" -// RVA23S64: "-target-feature" "+ssu64xl" -// RVA23S64: "-target-feature" "+svade" -// RVA23S64: "-target-feature" "+svbare" -// RVA23S64: "-target-feature" "+svinval" -// RVA23S64: "-target-feature" "+svnapot" -// RVA23S64: "-target-feature" "+svpbmt" - -// RUN: %clang -### -c %s 2>&1 -march=rvb23u64 -menable-experimental-extensions | FileCheck -check-prefix=RVB23U64 %s -// RVB23U64: "-target-feature" "+m" -// RVB23U64: "-target-feature" "+a" -// RVB23U64: "-target-feature" "+f" -// RVB23U64: "-target-feature" "+d" -// RVB23U64: "-target-feature" "+c" -// RVB23U64: "-target-feature" "+zic64b" -// RVB23U64: "-target-feature" "+zicbom" -// RVB23U64: "-target-feature" "+zicbop" -// RVB23U64: "-target-feature" "+zicboz" -// RVB23U64: "-target-feature" "+ziccamoa" -// RVB23U64: "-target-feature" "+ziccif" -// RVB23U64: "-target-feature" "+zicclsm" -// RVB23U64: "-target-feature" "+ziccrse" -// RVB23U64: "-target-feature" "+zicntr" -// RVB23U64: "-target-feature" "+zicond" -// RVB23U64: "-target-feature" "+zicsr" -// RVB23U64: "-target-feature" "+zihintntl" -// RVB23U64: "-target-feature" "+zihintpause" -// RVB23U64: "-target-feature" "+zihpm" -// RVB23U64: "-target-feature" "+experimental-zimop" -// RVB23U64: "-target-feature" "+za64rs" -// RVB23U64: "-target-feature" "+zawrs" -// RVB23U64: "-target-feature" "+zfa" -// RVB23U64: "-target-feature" "+zcb" -// RVB23U64: "-target-feature" "+experimental-zcmop" -// RVB23U64: "-target-feature" "+zba" -// RVB23U64: "-target-feature" "+zbb" -// RVB23U64: "-target-feature" "+zbs" -// RVB23U64: "-target-feature" "+zkt" - -// RUN: %clang -### -c %s 2>&1 -march=rvb23s64 -menable-experimental-extensions | FileCheck -check-prefix=RVB23S64 %s -// RVB23S64: "-target-feature" "+m" -// RVB23S64: "-target-feature" "+a" -// RVB23S64: "-target-feature" "+f" -// RVB23S64: "-target-feature" "+d" -// RVB23S64: "-target-feature" "+c" -// RVB23S64: "-target-feature" "+zic64b" -// RVB23S64: "-target-feature" "+zicbom" -// RVB23S64: "-target-feature" "+zicbop" -// RVB23S64: "-target-feature" "+zicboz" -// RVB23S64: "-target-feature" "+ziccamoa" -// RVB23S64: "-target-feature" "+ziccif" -// RVB23S64: "-target-feature" "+zicclsm" -// RVB23S64: "-target-feature" "+ziccrse" -// RVB23S64: "-target-feature" "+zicntr" -// RVB23S64: "-target-feature" "+zicond" -// RVB23S64: "-target-feature" "+zicsr" -// RVB23S64: "-target-feature" "+zifencei" -// RVB23S64: "-target-feature" "+zihintntl" -// RVB23S64: "-target-feature" "+zihintpause" -// RVB23S64: "-target-feature" "+zihpm" -// RVB23S64: "-target-feature" "+experimental-zimop" -// RVB23S64: "-target-feature" "+za64rs" -// RVB23S64: "-target-feature" "+zawrs" -// RVB23S64: "-target-feature" "+zfa" -// RVB23S64: "-target-feature" "+zcb" -// RVB23S64: "-target-feature" "+experimental-zcmop" -// RVB23S64: "-target-feature" "+zba" -// RVB23S64: "-target-feature" "+zbb" -// RVB23S64: "-target-feature" "+zbs" -// RVB23S64: "-target-feature" "+zkt" -// RVB23S64: "-target-feature" "+ssccptr" -// RVB23S64: "-target-feature" "+sscofpmf" -// RVB23S64: "-target-feature" "+sscounterenw" -// RVB23S64: "-target-feature" "+sstc" -// RVB23S64: "-target-feature" "+sstvala" -// RVB23S64: "-target-feature" "+sstvecd" -// RVB23S64: "-target-feature" "+ssu64xl" -// RVB23S64: "-target-feature" "+svade" -// RVB23S64: "-target-feature" "+svbare" -// RVB23S64: "-target-feature" "+svinval" -// RVB23S64: "-target-feature" "+svnapot" -// RVB23S64: "-target-feature" "+svpbmt" - -// RUN: %clang -### -c %s 2>&1 -march=rvm23u32 -menable-experimental-extensions | FileCheck -check-prefix=RVM23U32 %s -// RVM23U32: "-target-feature" "+m" -// RVM23U32: "-target-feature" "+zicbop" -// RVM23U32: "-target-feature" "+zicond" -// RVM23U32: "-target-feature" "+zicsr" -// RVM23U32: "-target-feature" "+zihintntl" -// RVM23U32: "-target-feature" "+zihintpause" -// RVM23U32: "-target-feature" "+experimental-zimop" -// RVM23U32: "-target-feature" "+zce" -// RVM23U32: "-target-feature" "+experimental-zcmop" -// RVM23U32: "-target-feature" "+zba" -// RVM23U32: "-target-feature" "+zbb" -// RVM23U32: "-target-feature" "+zbs" - -// RUN: %clang -### -c %s 2>&1 -march=rva22u64_zfa | FileCheck -check-prefix=PROFILE-WITH-ADDITIONAL %s -// PROFILE-WITH-ADDITIONAL: "-target-feature" "+m" -// PROFILE-WITH-ADDITIONAL: "-target-feature" "+a" -// PROFILE-WITH-ADDITIONAL: "-target-feature" "+f" -// PROFILE-WITH-ADDITIONAL: "-target-feature" "+d" -// PROFILE-WITH-ADDITIONAL: "-target-feature" "+c" -// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zicbom" -// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zicbop" -// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zicboz" -// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zihintpause" -// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zfa" -// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zfhmin" -// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zba" -// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zbb" -// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zbs" -// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zkt" - -// RUN: not %clang -### -c %s 2>&1 -march=rva19u64_zfa | FileCheck -check-prefix=INVALID-PROFILE %s -// INVALID-PROFILE: error: invalid arch name 'rva19u64_zfa', unsupported profile - -// RUN: not %clang -### -c %s 2>&1 -march=rva22u64zfa | FileCheck -check-prefix=INVALID-ADDITIONAL %s -// INVALID-ADDITIONAL: error: invalid arch name 'rva22u64zfa', additional extensions must be after separator '_' diff --git a/llvm/lib/Support/RISCVISAInfo.cpp b/llvm/lib/Support/RISCVISAInfo.cpp index 67e6e5b962b1..39235ace4724 100644 --- a/llvm/lib/Support/RISCVISAInfo.cpp +++ b/llvm/lib/Support/RISCVISAInfo.cpp @@ -36,11 +36,6 @@ struct RISCVSupportedExtension { } }; -struct RISCVProfile { - StringLiteral Name; - StringLiteral MArch; -}; - } // end anonymous namespace static constexpr StringLiteral AllStdExts = "mafdqlcbkjtpvnh"; @@ -249,42 +244,6 @@ static const RISCVSupportedExtension SupportedExperimentalExtensions[] = { }; // clang-format on -static constexpr RISCVProfile SupportedProfiles[] = { - {"rvi20u32", "rv32i"}, - {"rvi20u64", "rv64i"}, - {"rva20u64", "rv64imafdc_ziccamoa_ziccif_zicclsm_ziccrse_zicntr_za128rs"}, - {"rva20s64", "rv64imafdc_ziccamoa_ziccif_zicclsm_ziccrse_zicntr_zifencei_" - "za128rs_ssccptr_sstvala_sstvecd_svade_svbare"}, - {"rva22u64", - "rv64imafdc_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_zicclsm_ziccrse_" - "zicntr_zihintpause_zihpm_za64rs_zfhmin_zba_zbb_zbs_zkt"}, - {"rva22s64", - "rv64imafdc_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_zicclsm_ziccrse_" - "zicntr_zifencei_zihintpause_zihpm_za64rs_zfhmin_zba_zbb_zbs_zkt_ssccptr_" - "sscounterenw_sstvala_sstvecd_svade_svbare_svinval_svpbmt"}, - {"rva23u64", - "rv64imafdcv_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_zicclsm_ziccrse_" - "zicntr_zicond_zihintntl_zihintpause_zihpm_zimop0p1_za64rs_zawrs_zfa_" - "zfhmin_zcb_zcmop0p2_zba_zbb_zbs_zkt_zvbb_zvfhmin_zvkt"}, - {"rva23s64", - "rv64imafdcvh_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_zicclsm_ziccrse_" - "zicntr_zicond_zifencei_zihintntl_zihintpause_zihpm_zimop0p1_za64rs_zawrs_" - "zfa_zfhmin_zcb_zcmop0p2_zba_zbb_zbs_zkt_zvbb_zvfhmin_zvkt_shcounterenw_" - "shgatpa_shtvala_shvsatpa_shvstvala_shvstvecd_ssccptr_sscofpmf_" - "sscounterenw_ssnpm0p8_ssstateen_sstc_sstvala_sstvecd_ssu64xl_svade_" - "svbare_svinval_svnapot_svpbmt"}, - {"rvb23u64", "rv64imafdc_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_" - "zicclsm_ziccrse_zicntr_zicond_zihintntl_zihintpause_zihpm_" - "zimop0p1_za64rs_zawrs_zfa_zcb_zcmop0p2_zba_zbb_zbs_zkt"}, - {"rvb23s64", - "rv64imafdc_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_zicclsm_ziccrse_" - "zicntr_zicond_zifencei_zihintntl_zihintpause_zihpm_zimop0p1_za64rs_zawrs_" - "zfa_zcb_zcmop0p2_zba_zbb_zbs_zkt_ssccptr_sscofpmf_sscounterenw_sstc_" - "sstvala_sstvecd_ssu64xl_svade_svbare_svinval_svnapot_svpbmt"}, - {"rvm23u32", "rv32im_zicbop_zicond_zicsr_zihintntl_zihintpause_zimop0p1_" - "zca_zcb_zce_zcmop0p2_zcmp_zcmt_zba_zbb_zbs"}, -}; - static void verifyTables() { #ifndef NDEBUG static std::atomic TableChecked(false); @@ -898,29 +857,6 @@ RISCVISAInfo::parseArchString(StringRef Arch, bool EnableExperimentalExtension, "string must be lowercase"); } - if (Arch.starts_with("rvi") || Arch.starts_with("rva") || - Arch.starts_with("rvb") || Arch.starts_with("rvm")) { - const auto *FoundProfile = - llvm::find_if(SupportedProfiles, [Arch](const RISCVProfile &Profile) { - return Arch.starts_with(Profile.Name); - }); - - if (FoundProfile == std::end(SupportedProfiles)) - return createStringError(errc::invalid_argument, "unsupported profile"); - - std::string NewArch = FoundProfile->MArch.str(); - StringRef ArchWithoutProfile = Arch.substr(FoundProfile->Name.size()); - if (!ArchWithoutProfile.empty()) { - if (!ArchWithoutProfile.starts_with("_")) - return createStringError( - errc::invalid_argument, - "additional extensions must be after separator '_'"); - NewArch += ArchWithoutProfile.str(); - } - return parseArchString(NewArch, EnableExperimentalExtension, - ExperimentalExtensionVersionCheck, IgnoreUnknown); - } - bool HasRV64 = Arch.starts_with("rv64"); // ISA string must begin with rv32 or rv64. if (!(Arch.starts_with("rv32") || HasRV64) || (Arch.size() < 5)) { -- GitLab From a62441d4bb6bd0cd8eccab8c5692340c5a2c60bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20=C3=81lvarez=20Ayll=C3=B3n?= Date: Fri, 22 Mar 2024 11:50:34 +0100 Subject: [PATCH 036/306] [clang][analyzer][NFC] UnixAPIMisuseChecker inherits from Checker (#83027) --- .../Checkers/UnixAPIChecker.cpp | 60 +++++++++---------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/clang/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp index 19f1ca2dc824..599e5e6cedc6 100644 --- a/clang/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp @@ -17,6 +17,7 @@ #include "clang/StaticAnalyzer/Core/BugReporter/CommonBugCategories.h" #include "clang/StaticAnalyzer/Core/Checker.h" #include "clang/StaticAnalyzer/Core/CheckerManager.h" +#include "clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h" #include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h" #include "clang/StaticAnalyzer/Core/PathSensitive/CheckerHelpers.h" #include "llvm/ADT/STLExtras.h" @@ -41,8 +42,7 @@ enum class OpenVariant { namespace { class UnixAPIMisuseChecker - : public Checker, - check::ASTDecl> { + : public Checker> { const BugType BT_open{this, "Improper use of 'open'", categories::UnixAPI}; const BugType BT_pthreadOnce{this, "Improper use of 'pthread_once'", categories::UnixAPI}; @@ -52,14 +52,14 @@ public: void checkASTDecl(const TranslationUnitDecl *TU, AnalysisManager &Mgr, BugReporter &BR) const; - void checkPreStmt(const CallExpr *CE, CheckerContext &C) const; + void checkPreCall(const CallEvent &Call, CheckerContext &C) const; - void CheckOpen(CheckerContext &C, const CallExpr *CE) const; - void CheckOpenAt(CheckerContext &C, const CallExpr *CE) const; - void CheckPthreadOnce(CheckerContext &C, const CallExpr *CE) const; + void CheckOpen(CheckerContext &C, const CallEvent &Call) const; + void CheckOpenAt(CheckerContext &C, const CallEvent &Call) const; + void CheckPthreadOnce(CheckerContext &C, const CallEvent &Call) const; - void CheckOpenVariant(CheckerContext &C, - const CallExpr *CE, OpenVariant Variant) const; + void CheckOpenVariant(CheckerContext &C, const CallEvent &Call, + OpenVariant Variant) const; void ReportOpenBug(CheckerContext &C, ProgramStateRef State, const char *Msg, SourceRange SR) const; @@ -113,9 +113,9 @@ void UnixAPIMisuseChecker::checkASTDecl(const TranslationUnitDecl *TU, // "open" (man 2 open) //===----------------------------------------------------------------------===/ -void UnixAPIMisuseChecker::checkPreStmt(const CallExpr *CE, +void UnixAPIMisuseChecker::checkPreCall(const CallEvent &Call, CheckerContext &C) const { - const FunctionDecl *FD = C.getCalleeDecl(CE); + const FunctionDecl *FD = dyn_cast_if_present(Call.getDecl()); if (!FD || FD->getKind() != Decl::Function) return; @@ -130,13 +130,13 @@ void UnixAPIMisuseChecker::checkPreStmt(const CallExpr *CE, return; if (FName == "open") - CheckOpen(C, CE); + CheckOpen(C, Call); else if (FName == "openat") - CheckOpenAt(C, CE); + CheckOpenAt(C, Call); else if (FName == "pthread_once") - CheckPthreadOnce(C, CE); + CheckPthreadOnce(C, Call); } void UnixAPIMisuseChecker::ReportOpenBug(CheckerContext &C, ProgramStateRef State, @@ -152,17 +152,17 @@ void UnixAPIMisuseChecker::ReportOpenBug(CheckerContext &C, } void UnixAPIMisuseChecker::CheckOpen(CheckerContext &C, - const CallExpr *CE) const { - CheckOpenVariant(C, CE, OpenVariant::Open); + const CallEvent &Call) const { + CheckOpenVariant(C, Call, OpenVariant::Open); } void UnixAPIMisuseChecker::CheckOpenAt(CheckerContext &C, - const CallExpr *CE) const { - CheckOpenVariant(C, CE, OpenVariant::OpenAt); + const CallEvent &Call) const { + CheckOpenVariant(C, Call, OpenVariant::OpenAt); } void UnixAPIMisuseChecker::CheckOpenVariant(CheckerContext &C, - const CallExpr *CE, + const CallEvent &Call, OpenVariant Variant) const { // The index of the argument taking the flags open flags (O_RDONLY, // O_WRONLY, O_CREAT, etc.), @@ -191,11 +191,11 @@ void UnixAPIMisuseChecker::CheckOpenVariant(CheckerContext &C, ProgramStateRef state = C.getState(); - if (CE->getNumArgs() < MinArgCount) { + if (Call.getNumArgs() < MinArgCount) { // The frontend should issue a warning for this case. Just return. return; - } else if (CE->getNumArgs() == MaxArgCount) { - const Expr *Arg = CE->getArg(CreateModeArgIndex); + } else if (Call.getNumArgs() == MaxArgCount) { + const Expr *Arg = Call.getArgExpr(CreateModeArgIndex); QualType QT = Arg->getType(); if (!QT->isIntegerType()) { SmallString<256> SBuf; @@ -209,7 +209,7 @@ void UnixAPIMisuseChecker::CheckOpenVariant(CheckerContext &C, Arg->getSourceRange()); return; } - } else if (CE->getNumArgs() > MaxArgCount) { + } else if (Call.getNumArgs() > MaxArgCount) { SmallString<256> SBuf; llvm::raw_svector_ostream OS(SBuf); OS << "Call to '" << VariantName << "' with more than " << MaxArgCount @@ -217,7 +217,7 @@ void UnixAPIMisuseChecker::CheckOpenVariant(CheckerContext &C, ReportOpenBug(C, state, SBuf.c_str(), - CE->getArg(MaxArgCount)->getSourceRange()); + Call.getArgExpr(MaxArgCount)->getSourceRange()); return; } @@ -226,8 +226,8 @@ void UnixAPIMisuseChecker::CheckOpenVariant(CheckerContext &C, } // Now check if oflags has O_CREAT set. - const Expr *oflagsEx = CE->getArg(FlagsArgIndex); - const SVal V = C.getSVal(oflagsEx); + const Expr *oflagsEx = Call.getArgExpr(FlagsArgIndex); + const SVal V = Call.getArgSVal(FlagsArgIndex); if (!isa(V)) { // The case where 'V' can be a location can only be due to a bad header, // so in this case bail out. @@ -253,7 +253,7 @@ void UnixAPIMisuseChecker::CheckOpenVariant(CheckerContext &C, if (!(trueState && !falseState)) return; - if (CE->getNumArgs() < MaxArgCount) { + if (Call.getNumArgs() < MaxArgCount) { SmallString<256> SBuf; llvm::raw_svector_ostream OS(SBuf); OS << "Call to '" << VariantName << "' requires a " @@ -271,18 +271,18 @@ void UnixAPIMisuseChecker::CheckOpenVariant(CheckerContext &C, //===----------------------------------------------------------------------===// void UnixAPIMisuseChecker::CheckPthreadOnce(CheckerContext &C, - const CallExpr *CE) const { + const CallEvent &Call) const { // This is similar to 'CheckDispatchOnce' in the MacOSXAPIChecker. // They can possibly be refactored. - if (CE->getNumArgs() < 1) + if (Call.getNumArgs() < 1) return; // Check if the first argument is stack allocated. If so, issue a warning // because that's likely to be bad news. ProgramStateRef state = C.getState(); - const MemRegion *R = C.getSVal(CE->getArg(0)).getAsRegion(); + const MemRegion *R = Call.getArgSVal(0).getAsRegion(); if (!R || !isa(R->getMemorySpace())) return; @@ -304,7 +304,7 @@ void UnixAPIMisuseChecker::CheckPthreadOnce(CheckerContext &C, auto report = std::make_unique(BT_pthreadOnce, os.str(), N); - report->addRange(CE->getArg(0)->getSourceRange()); + report->addRange(Call.getArgExpr(0)->getSourceRange()); C.emitReport(std::move(report)); } -- GitLab From 730ca47a0cc7380def6df1d25b30c1378fd8bf14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20=C3=81lvarez=20Ayll=C3=B3n?= Date: Fri, 22 Mar 2024 11:50:34 +0100 Subject: [PATCH 037/306] [clang][analyzer] Model getline/getdelim preconditions and evaluation (#83027) According to POSIX 2018. 1. lineptr, n and stream can not be NULL. 2. If *n is non-zero, *lineptr must point to a region of at least *n bytes, or be a NULL pointer. Additionally, if *lineptr is not NULL, *n must not be undefined. --- .../Core/PathSensitive/CheckerHelpers.h | 4 +- .../StaticAnalyzer/Checkers/MallocChecker.cpp | 8 +- .../StaticAnalyzer/Checkers/StreamChecker.cpp | 23 +- .../Checkers/UnixAPIChecker.cpp | 132 ++++++- .../StaticAnalyzer/Core/CheckerHelpers.cpp | 5 +- clang/test/Analysis/getline-unixapi.c | 322 ++++++++++++++++++ clang/test/Analysis/stream.c | 73 ++++ 7 files changed, 554 insertions(+), 13 deletions(-) create mode 100644 clang/test/Analysis/getline-unixapi.c diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CheckerHelpers.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CheckerHelpers.h index 60421e5437d8..d053a9718912 100644 --- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CheckerHelpers.h +++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CheckerHelpers.h @@ -15,7 +15,6 @@ #include "ProgramState_Fwd.h" #include "SVals.h" - #include "clang/AST/OperationKinds.h" #include "clang/AST/Stmt.h" #include "clang/Basic/OperatorKinds.h" @@ -113,8 +112,7 @@ public: OperatorKind operationKindFromOverloadedOperator(OverloadedOperatorKind OOK, bool IsBinary); -std::optional getPointeeDefVal(SVal PtrSVal, - ProgramStateRef State); +std::optional getPointeeVal(SVal PtrSVal, ProgramStateRef State); } // namespace ento diff --git a/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp index 03cb7696707f..c2d96f592609 100644 --- a/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp @@ -1441,7 +1441,7 @@ void MallocChecker::preGetdelim(const CallEvent &Call, return; ProgramStateRef State = C.getState(); - const auto LinePtr = getPointeeDefVal(Call.getArgSVal(0), State); + const auto LinePtr = getPointeeVal(Call.getArgSVal(0), State); if (!LinePtr) return; @@ -1470,8 +1470,10 @@ void MallocChecker::checkGetdelim(const CallEvent &Call, SValBuilder &SVB = C.getSValBuilder(); - const auto LinePtr = getPointeeDefVal(Call.getArgSVal(0), State); - const auto Size = getPointeeDefVal(Call.getArgSVal(1), State); + const auto LinePtr = + getPointeeVal(Call.getArgSVal(0), State)->getAs(); + const auto Size = + getPointeeVal(Call.getArgSVal(1), State)->getAs(); if (!LinePtr || !Size || !LinePtr->getAsRegion()) return; diff --git a/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp index 10972158f398..902c42a2799b 100644 --- a/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp @@ -1200,10 +1200,25 @@ void StreamChecker::evalGetdelim(const FnDescription *Desc, // Add transition for the successful state. NonLoc RetVal = makeRetVal(C, E.CE).castAs(); - ProgramStateRef StateNotFailed = - State->BindExpr(E.CE, C.getLocationContext(), RetVal); + ProgramStateRef StateNotFailed = E.bindReturnValue(State, C, RetVal); StateNotFailed = E.assumeBinOpNN(StateNotFailed, BO_GE, RetVal, E.getZeroVal(Call)); + + // On success, a buffer is allocated. + auto NewLinePtr = getPointeeVal(Call.getArgSVal(0), State); + if (NewLinePtr && isa(*NewLinePtr)) + StateNotFailed = StateNotFailed->assume( + NewLinePtr->castAs(), true); + + // The buffer size `*n` must be enough to hold the whole line, and + // greater than the return value, since it has to account for '\0'. + SVal SizePtrSval = Call.getArgSVal(1); + auto NVal = getPointeeVal(SizePtrSval, State); + if (NVal && isa(*NVal)) { + StateNotFailed = E.assumeBinOpNN(StateNotFailed, BO_GT, + NVal->castAs(), RetVal); + StateNotFailed = E.bindReturnValue(StateNotFailed, C, RetVal); + } if (!StateNotFailed) return; C.addTransition(StateNotFailed); @@ -1217,6 +1232,10 @@ void StreamChecker::evalGetdelim(const FnDescription *Desc, E.isStreamEof() ? ErrorFEof : ErrorFEof | ErrorFError; StateFailed = E.setStreamState( StateFailed, StreamState::getOpened(Desc, NewES, !NewES.isFEof())); + // On failure, the content of the buffer is undefined. + if (auto NewLinePtr = getPointeeVal(Call.getArgSVal(0), State)) + StateFailed = StateFailed->bindLoc(*NewLinePtr, UndefinedVal(), + C.getLocationContext()); C.addTransition(StateFailed, E.getFailureNoteTag(this, C)); } diff --git a/clang/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp index 599e5e6cedc6..da2d16ca9b5d 100644 --- a/clang/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp @@ -20,6 +20,7 @@ #include "clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h" #include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h" #include "clang/StaticAnalyzer/Core/PathSensitive/CheckerHelpers.h" +#include "clang/StaticAnalyzer/Core/PathSensitive/DynamicExtent.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/StringExtras.h" @@ -44,10 +45,23 @@ namespace { class UnixAPIMisuseChecker : public Checker> { const BugType BT_open{this, "Improper use of 'open'", categories::UnixAPI}; + const BugType BT_getline{this, "Improper use of getdelim", + categories::UnixAPI}; const BugType BT_pthreadOnce{this, "Improper use of 'pthread_once'", categories::UnixAPI}; + const BugType BT_ArgumentNull{this, "NULL pointer", categories::UnixAPI}; mutable std::optional Val_O_CREAT; + ProgramStateRef + EnsurePtrNotNull(SVal PtrVal, const Expr *PtrExpr, CheckerContext &C, + ProgramStateRef State, const StringRef PtrDescr, + std::optional> BT = + std::nullopt) const; + + ProgramStateRef EnsureGetdelimBufferAndSizeCorrect( + SVal LinePtrPtrSVal, SVal SizePtrSVal, const Expr *LinePtrPtrExpr, + const Expr *SizePtrExpr, CheckerContext &C, ProgramStateRef State) const; + public: void checkASTDecl(const TranslationUnitDecl *TU, AnalysisManager &Mgr, BugReporter &BR) const; @@ -56,6 +70,7 @@ public: void CheckOpen(CheckerContext &C, const CallEvent &Call) const; void CheckOpenAt(CheckerContext &C, const CallEvent &Call) const; + void CheckGetDelim(CheckerContext &C, const CallEvent &Call) const; void CheckPthreadOnce(CheckerContext &C, const CallEvent &Call) const; void CheckOpenVariant(CheckerContext &C, const CallEvent &Call, @@ -95,6 +110,30 @@ private: } // end anonymous namespace +ProgramStateRef UnixAPIMisuseChecker::EnsurePtrNotNull( + SVal PtrVal, const Expr *PtrExpr, CheckerContext &C, ProgramStateRef State, + const StringRef PtrDescr, + std::optional> BT) const { + const auto Ptr = PtrVal.getAs(); + if (!Ptr) + return State; + + const auto [PtrNotNull, PtrNull] = State->assume(*Ptr); + if (!PtrNotNull && PtrNull) { + if (ExplodedNode *N = C.generateErrorNode(PtrNull)) { + auto R = std::make_unique( + BT.value_or(std::cref(BT_ArgumentNull)), + (PtrDescr + " pointer might be NULL.").str(), N); + if (PtrExpr) + bugreporter::trackExpressionValue(N, PtrExpr, *R); + C.emitReport(std::move(R)); + } + return nullptr; + } + + return PtrNotNull; +} + void UnixAPIMisuseChecker::checkASTDecl(const TranslationUnitDecl *TU, AnalysisManager &Mgr, BugReporter &) const { @@ -137,6 +176,9 @@ void UnixAPIMisuseChecker::checkPreCall(const CallEvent &Call, else if (FName == "pthread_once") CheckPthreadOnce(C, Call); + + else if (is_contained({"getdelim", "getline"}, FName)) + CheckGetDelim(C, Call); } void UnixAPIMisuseChecker::ReportOpenBug(CheckerContext &C, ProgramStateRef State, @@ -215,8 +257,7 @@ void UnixAPIMisuseChecker::CheckOpenVariant(CheckerContext &C, OS << "Call to '" << VariantName << "' with more than " << MaxArgCount << " arguments"; - ReportOpenBug(C, state, - SBuf.c_str(), + ReportOpenBug(C, state, SBuf.c_str(), Call.getArgExpr(MaxArgCount)->getSourceRange()); return; } @@ -266,6 +307,93 @@ void UnixAPIMisuseChecker::CheckOpenVariant(CheckerContext &C, } } +//===----------------------------------------------------------------------===// +// getdelim and getline +//===----------------------------------------------------------------------===// + +ProgramStateRef UnixAPIMisuseChecker::EnsureGetdelimBufferAndSizeCorrect( + SVal LinePtrPtrSVal, SVal SizePtrSVal, const Expr *LinePtrPtrExpr, + const Expr *SizePtrExpr, CheckerContext &C, ProgramStateRef State) const { + static constexpr llvm::StringLiteral SizeGreaterThanBufferSize = + "The buffer from the first argument is smaller than the size " + "specified by the second parameter"; + static constexpr llvm::StringLiteral SizeUndef = + "The buffer from the first argument is not NULL, but the size specified " + "by the second parameter is undefined."; + + auto EmitBugReport = [this, &C, SizePtrExpr, LinePtrPtrExpr]( + ProgramStateRef BugState, StringRef ErrMsg) { + if (ExplodedNode *N = C.generateErrorNode(BugState)) { + auto R = std::make_unique(BT_getline, ErrMsg, N); + bugreporter::trackExpressionValue(N, SizePtrExpr, *R); + bugreporter::trackExpressionValue(N, LinePtrPtrExpr, *R); + C.emitReport(std::move(R)); + } + }; + + // We have a pointer to a pointer to the buffer, and a pointer to the size. + // We want what they point at. + auto LinePtrSVal = getPointeeVal(LinePtrPtrSVal, State)->getAs(); + auto NSVal = getPointeeVal(SizePtrSVal, State); + if (!LinePtrSVal || !NSVal || NSVal->isUnknown()) + return nullptr; + + assert(LinePtrPtrExpr && SizePtrExpr); + + const auto [LinePtrNotNull, LinePtrNull] = State->assume(*LinePtrSVal); + if (LinePtrNotNull && !LinePtrNull) { + // If `*lineptr` is not null, but `*n` is undefined, there is UB. + if (NSVal->isUndef()) { + EmitBugReport(LinePtrNotNull, SizeUndef); + return nullptr; + } + + // If it is defined, and known, its size must be less than or equal to + // the buffer size. + auto NDefSVal = NSVal->getAs(); + auto &SVB = C.getSValBuilder(); + auto LineBufSize = + getDynamicExtent(LinePtrNotNull, LinePtrSVal->getAsRegion(), SVB); + auto LineBufSizeGtN = SVB.evalBinOp(LinePtrNotNull, BO_GE, LineBufSize, + *NDefSVal, SVB.getConditionType()) + .getAs(); + if (!LineBufSizeGtN) + return LinePtrNotNull; + if (auto LineBufSizeOk = LinePtrNotNull->assume(*LineBufSizeGtN, true)) + return LineBufSizeOk; + + EmitBugReport(LinePtrNotNull, SizeGreaterThanBufferSize); + return nullptr; + } + return State; +} + +void UnixAPIMisuseChecker::CheckGetDelim(CheckerContext &C, + const CallEvent &Call) const { + ProgramStateRef State = C.getState(); + + // The parameter `n` must not be NULL. + SVal SizePtrSval = Call.getArgSVal(1); + State = EnsurePtrNotNull(SizePtrSval, Call.getArgExpr(1), C, State, "Size"); + if (!State) + return; + + // The parameter `lineptr` must not be NULL. + SVal LinePtrPtrSVal = Call.getArgSVal(0); + State = + EnsurePtrNotNull(LinePtrPtrSVal, Call.getArgExpr(0), C, State, "Line"); + if (!State) + return; + + State = EnsureGetdelimBufferAndSizeCorrect(LinePtrPtrSVal, SizePtrSval, + Call.getArgExpr(0), + Call.getArgExpr(1), C, State); + if (!State) + return; + + C.addTransition(State); +} + //===----------------------------------------------------------------------===// // pthread_once //===----------------------------------------------------------------------===// diff --git a/clang/lib/StaticAnalyzer/Core/CheckerHelpers.cpp b/clang/lib/StaticAnalyzer/Core/CheckerHelpers.cpp index 364c87e910b7..d7137a915b3d 100644 --- a/clang/lib/StaticAnalyzer/Core/CheckerHelpers.cpp +++ b/clang/lib/StaticAnalyzer/Core/CheckerHelpers.cpp @@ -183,10 +183,9 @@ OperatorKind operationKindFromOverloadedOperator(OverloadedOperatorKind OOK, } } -std::optional getPointeeDefVal(SVal PtrSVal, - ProgramStateRef State) { +std::optional getPointeeVal(SVal PtrSVal, ProgramStateRef State) { if (const auto *Ptr = PtrSVal.getAsRegion()) { - return State->getSVal(Ptr).getAs(); + return State->getSVal(Ptr); } return std::nullopt; } diff --git a/clang/test/Analysis/getline-unixapi.c b/clang/test/Analysis/getline-unixapi.c new file mode 100644 index 000000000000..86635ed84997 --- /dev/null +++ b/clang/test/Analysis/getline-unixapi.c @@ -0,0 +1,322 @@ +// RUN: %clang_analyze_cc1 -analyzer-checker=core,unix,debug.ExprInspection -verify %s + +#include "Inputs/system-header-simulator.h" +#include "Inputs/system-header-simulator-for-malloc.h" +#include "Inputs/system-header-simulator-for-valist.h" + +void clang_analyzer_eval(int); +void clang_analyzer_dump_int(int); +void clang_analyzer_dump_ptr(void*); +void clang_analyzer_warnIfReached(); + +void test_getline_null_lineptr() { + FILE *F1 = tmpfile(); + if (!F1) + return; + + char **buffer = NULL; + size_t n = 0; + getline(buffer, &n, F1); // expected-warning {{Line pointer might be NULL}} + fclose(F1); +} + +void test_getline_null_size() { + FILE *F1 = tmpfile(); + if (!F1) + return; + char *buffer = NULL; + getline(&buffer, NULL, F1); // expected-warning {{Size pointer might be NULL}} + fclose(F1); +} + +void test_getline_null_buffer_size_gt0() { + FILE *F1 = tmpfile(); + if (!F1) + return; + char *buffer = NULL; + size_t n = 8; + getline(&buffer, &n, F1); // ok since posix 2018 + free(buffer); + fclose(F1); +} + +void test_getline_null_buffer_size_gt0_2(size_t n) { + FILE *F1 = tmpfile(); + if (!F1) + return; + char *buffer = NULL; + if (n > 0) { + getline(&buffer, &n, F1); // ok since posix 2018 + } + free(buffer); + fclose(F1); +} + +void test_getline_null_buffer_unknown_size(size_t n) { + FILE *F1 = tmpfile(); + if (!F1) + return; + char *buffer = NULL; + + getline(&buffer, &n, F1); // ok + fclose(F1); + free(buffer); +} + +void test_getline_null_buffer_undef_size() { + FILE *F1 = tmpfile(); + if (!F1) + return; + + char *buffer = NULL; + size_t n; + + getline(&buffer, &n, F1); // ok since posix 2018 + fclose(F1); + free(buffer); +} + +void test_getline_buffer_size_0() { + FILE *F1 = tmpfile(); + if (!F1) + return; + + char *buffer = malloc(10); + size_t n = 0; + if (buffer != NULL) + getline(&buffer, &n, F1); // ok, the buffer is enough for 0 character + fclose(F1); + free(buffer); +} + +void test_getline_buffer_bad_size() { + FILE *F1 = tmpfile(); + if (!F1) + return; + + char *buffer = malloc(10); + size_t n = 100; + if (buffer != NULL) + getline(&buffer, &n, F1); // expected-warning {{The buffer from the first argument is smaller than the size specified by the second parameter}} + fclose(F1); + free(buffer); +} + +void test_getline_buffer_smaller_size() { + FILE *F1 = tmpfile(); + if (!F1) + return; + + char *buffer = malloc(100); + size_t n = 10; + if (buffer != NULL) + getline(&buffer, &n, F1); // ok, there is enough space for 10 characters + fclose(F1); + free(buffer); +} + +void test_getline_buffer_undef_size() { + FILE *F1 = tmpfile(); + if (!F1) + return; + + char *buffer = malloc(100); + size_t n; + if (buffer != NULL) + getline(&buffer, &n, F1); // expected-warning {{The buffer from the first argument is not NULL, but the size specified by the second parameter is undefined}} + fclose(F1); + free(buffer); +} + + +void test_getline_null_buffer() { + FILE *F1 = tmpfile(); + if (!F1) + return; + char *buffer = NULL; + size_t n = 0; + ssize_t r = getline(&buffer, &n, F1); + // getline returns -1 on failure, number of char reads on success (>= 0) + if (r < -1) { + clang_analyzer_warnIfReached(); // must not happen + } else { + // The buffer could be allocated both on failure and success + clang_analyzer_dump_int(n); // expected-warning {{conj_$}} + clang_analyzer_dump_ptr(buffer); // expected-warning {{conj_$}} + } + free(buffer); + fclose(F1); +} + +void test_getdelim_null_size() { + FILE *F1 = tmpfile(); + if (!F1) + return; + char *buffer = NULL; + getdelim(&buffer, NULL, ',', F1); // expected-warning {{Size pointer might be NULL}} + fclose(F1); +} + +void test_getdelim_null_buffer_size_gt0() { + FILE *F1 = tmpfile(); + if (!F1) + return; + char *buffer = NULL; + size_t n = 8; + getdelim(&buffer, &n, ';', F1); // ok since posix 2018 + free(buffer); + fclose(F1); +} + +void test_getdelim_null_buffer_size_gt0_2(size_t n) { + FILE *F1 = tmpfile(); + if (!F1) + return; + char *buffer = NULL; + if (n > 0) { + getdelim(&buffer, &n, ' ', F1); // ok since posix 2018 + } + free(buffer); + fclose(F1); +} + +void test_getdelim_null_buffer_unknown_size(size_t n) { + FILE *F1 = tmpfile(); + if (!F1) + return; + char *buffer = NULL; + getdelim(&buffer, &n, '-', F1); // ok + fclose(F1); + free(buffer); +} + +void test_getdelim_null_buffer() { + FILE *F1 = tmpfile(); + if (!F1) + return; + char *buffer = NULL; + size_t n = 0; + ssize_t r = getdelim(&buffer, &n, '\r', F1); + // getdelim returns -1 on failure, number of char reads on success (>= 0) + if (r < -1) { + clang_analyzer_warnIfReached(); // must not happen + } + else { + // The buffer could be allocated both on failure and success + clang_analyzer_dump_int(n); // expected-warning {{conj_$}} + clang_analyzer_dump_ptr(buffer); // expected-warning {{conj_$}} + } + free(buffer); + fclose(F1); +} + +void test_getline_while() { + FILE *file = fopen("file.txt", "r"); + if (file == NULL) { + return; + } + + char *line = NULL; + size_t len = 0; + ssize_t read; + + while ((read = getline(&line, &len, file)) != -1) { + printf("%s\n", line); + } + + free(line); + fclose(file); +} + +void test_getline_return_check() { + FILE *file = fopen("file.txt", "r"); + if (file == NULL) { + return; + } + + char *line = NULL; + size_t len = 0; + ssize_t r = getline(&line, &len, file); + + if (r != -1) { + if (line[0] == '\0') {} // ok + } + free(line); + fclose(file); +} + +void test_getline_clear_eof() { + FILE *file = fopen("file.txt", "r"); + if (file == NULL) { + return; + } + + size_t n = 10; + char *buffer = malloc(n); + ssize_t read = fread(buffer, n, 1, file); + if (feof(file)) { + clearerr(file); + getline(&buffer, &n, file); // ok + } + fclose(file); + free(buffer); +} + +void test_getline_not_null(char **buffer, size_t *size) { + FILE *file = fopen("file.txt", "r"); + if (file == NULL) { + return; + } + + getline(buffer, size, file); + fclose(file); + + if (size == NULL || buffer == NULL) { + clang_analyzer_warnIfReached(); // must not happen + } +} + +void test_getline_size_constraint(size_t size) { + FILE *file = fopen("file.txt", "r"); + if (file == NULL) { + return; + } + + size_t old_size = size; + char *buffer = malloc(10); + if (buffer != NULL) { + ssize_t r = getline(&buffer, &size, file); + if (r >= 0) { + // Since buffer has a size of 10, old_size must be less than or equal to 10. + // Otherwise, there would be UB. + clang_analyzer_eval(old_size <= 10); // expected-warning{{TRUE}} + } + } + fclose(file); + free(buffer); +} + +void test_getline_negative_buffer() { + FILE *file = fopen("file.txt", "r"); + if (file == NULL) { + return; + } + + char *buffer = NULL; + size_t n = -1; + getline(&buffer, &n, file); // ok since posix 2018 + free(buffer); + fclose(file); +} + +void test_getline_negative_buffer_2(char *buffer) { + FILE *file = fopen("file.txt", "r"); + if (file == NULL) { + return; + } + + size_t n = -1; + (void)getline(&buffer, &n, file); // ok + free(buffer); + fclose(file); +} diff --git a/clang/test/Analysis/stream.c b/clang/test/Analysis/stream.c index 7ba27740a937..ba5e66a4102e 100644 --- a/clang/test/Analysis/stream.c +++ b/clang/test/Analysis/stream.c @@ -4,6 +4,7 @@ // RUN: %clang_analyze_cc1 -triple=hexagon -analyzer-checker=core,alpha.unix.Stream,debug.ExprInspection -verify %s #include "Inputs/system-header-simulator.h" +#include "Inputs/system-header-simulator-for-malloc.h" #include "Inputs/system-header-simulator-for-valist.h" void clang_analyzer_eval(int); @@ -376,3 +377,75 @@ void fflush_on_open_failed_stream(void) { } fclose(F); } + +void getline_null_file() { + char *buffer = NULL; + size_t n = 0; + getline(&buffer, &n, NULL); // expected-warning {{Stream pointer might be NULL}} +} + +void getdelim_null_file() { + char *buffer = NULL; + size_t n = 0; + getdelim(&buffer, &n, '\n', NULL); // expected-warning {{Stream pointer might be NULL}} +} + +void getline_buffer_on_error() { + FILE *file = fopen("file.txt", "r"); + if (file == NULL) { + return; + } + + char *line = NULL; + size_t len = 0; + if (getline(&line, &len, file) == -1) { + if (line[0] == '\0') {} // expected-warning {{The left operand of '==' is a garbage value}} + } else { + if (line[0] == '\0') {} // no warning + } + + free(line); + fclose(file); +} + +void getline_ret_value() { + FILE *file = fopen("file.txt", "r"); + if (file == NULL) { + return; + } + + size_t n = 0; + char *buffer = NULL; + ssize_t r = getline(&buffer, &n, file); + + if (r > -1) { + // The return value does *not* include the terminating null byte. + // The buffer must be large enough to include it. + clang_analyzer_eval(n > r); // expected-warning{{TRUE}} + clang_analyzer_eval(buffer != NULL); // expected-warning{{TRUE}} + } + + fclose(file); + free(buffer); +} + + +void getline_buffer_size_negative() { + FILE *file = fopen("file.txt", "r"); + if (file == NULL) { + return; + } + + size_t n = -1; + clang_analyzer_eval((ssize_t)n >= 0); // expected-warning{{FALSE}} + char *buffer = NULL; + ssize_t r = getline(&buffer, &n, file); + + if (r > -1) { + clang_analyzer_eval((ssize_t)n > r); // expected-warning{{TRUE}} + clang_analyzer_eval(buffer != NULL); // expected-warning{{TRUE}} + } + + free(buffer); + fclose(file); +} -- GitLab From d8e5c0b4e546c73b2d10956a9517f1f2727702ae Mon Sep 17 00:00:00 2001 From: Farzon Lotfi <1802579+farzonl@users.noreply.github.com> Date: Fri, 22 Mar 2024 07:01:01 -0400 Subject: [PATCH 038/306] [DXIL] Complete abs lowering (#86158) This change completes #86155 - `DXIL.td` - lowering `fabs` intrinsic to the float dxil op. - `DXILIntrinsicExpansion.cpp` - Add intrinsic expansion for the abs case. --- llvm/lib/Target/DirectX/DXIL.td | 2 + .../Target/DirectX/DXILIntrinsicExpansion.cpp | 23 +++++++++++ llvm/test/CodeGen/DirectX/abs-vec.ll | 34 +++++++++++++++++ llvm/test/CodeGen/DirectX/abs.ll | 38 +++++++++++++++++++ llvm/test/CodeGen/DirectX/fabs.ll | 32 ++++++++++++++++ 5 files changed, 129 insertions(+) create mode 100644 llvm/test/CodeGen/DirectX/abs-vec.ll create mode 100644 llvm/test/CodeGen/DirectX/abs.ll create mode 100644 llvm/test/CodeGen/DirectX/fabs.ll diff --git a/llvm/lib/Target/DirectX/DXIL.td b/llvm/lib/Target/DirectX/DXIL.td index 36eb29d53766..ba7cd6ae9183 100644 --- a/llvm/lib/Target/DirectX/DXIL.td +++ b/llvm/lib/Target/DirectX/DXIL.td @@ -255,6 +255,8 @@ class DXILOpMapping; def IsInf : DXILOpMapping<9, isSpecialFloat, int_dx_isinf, "Determines if the specified value is infinite.", [llvm_i1_ty, llvm_halforfloat_ty]>; diff --git a/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp b/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp index 0db42bc0a0fb..b46564702c7a 100644 --- a/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp +++ b/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp @@ -33,6 +33,7 @@ using namespace llvm; static bool isIntrinsicExpansion(Function &F) { switch (F.getIntrinsicID()) { + case Intrinsic::abs: case Intrinsic::exp: case Intrinsic::dx_any: case Intrinsic::dx_clamp: @@ -46,6 +47,26 @@ static bool isIntrinsicExpansion(Function &F) { return false; } +static bool expandAbs(CallInst *Orig) { + Value *X = Orig->getOperand(0); + IRBuilder<> Builder(Orig->getParent()); + Builder.SetInsertPoint(Orig); + Type *Ty = X->getType(); + Type *EltTy = Ty->getScalarType(); + Constant *Zero = Ty->isVectorTy() + ? ConstantVector::getSplat( + ElementCount::getFixed( + cast(Ty)->getNumElements()), + ConstantInt::get(EltTy, 0)) + : ConstantInt::get(EltTy, 0); + auto *V = Builder.CreateSub(Zero, X); + auto *MaxCall = + Builder.CreateIntrinsic(Ty, Intrinsic::smax, {X, V}, nullptr, "dx.max"); + Orig->replaceAllUsesWith(MaxCall); + Orig->eraseFromParent(); + return true; +} + static bool expandIntegerDot(CallInst *Orig, Intrinsic::ID DotIntrinsic) { assert(DotIntrinsic == Intrinsic::dx_sdot || DotIntrinsic == Intrinsic::dx_udot); @@ -213,6 +234,8 @@ static bool expandClampIntrinsic(CallInst *Orig, Intrinsic::ID ClampIntrinsic) { static bool expandIntrinsic(Function &F, CallInst *Orig) { switch (F.getIntrinsicID()) { + case Intrinsic::abs: + return expandAbs(Orig); case Intrinsic::exp: return expandExpIntrinsic(Orig); case Intrinsic::dx_any: diff --git a/llvm/test/CodeGen/DirectX/abs-vec.ll b/llvm/test/CodeGen/DirectX/abs-vec.ll new file mode 100644 index 000000000000..1c40555eb390 --- /dev/null +++ b/llvm/test/CodeGen/DirectX/abs-vec.ll @@ -0,0 +1,34 @@ +; RUN: opt -S -dxil-intrinsic-expansion < %s | FileCheck %s + +; Make sure dxil operation function calls for abs are generated for int vectors. + +; CHECK-LABEL: abs_i16Vec2 +define noundef <2 x i16> @abs_i16Vec2(<2 x i16> noundef %a) #0 { +entry: +; CHECK: sub <2 x i16> zeroinitializer, %a +; CHECK: call <2 x i16> @llvm.smax.v2i16(<2 x i16> %a, <2 x i16> %{{.*}}) + %elt.abs = call <2 x i16> @llvm.abs.v2i16(<2 x i16> %a, i1 false) + ret <2 x i16> %elt.abs +} + +; CHECK-LABEL: abs_i32Vec3 +define noundef <3 x i32> @abs_i32Vec3(<3 x i32> noundef %a) #0 { +entry: +; CHECK: sub <3 x i32> zeroinitializer, %a +; CHECK: call <3 x i32> @llvm.smax.v3i32(<3 x i32> %a, <3 x i32> %{{.*}}) + %elt.abs = call <3 x i32> @llvm.abs.v3i32(<3 x i32> %a, i1 false) + ret <3 x i32> %elt.abs +} + +; CHECK-LABEL: abs_i64Vec4 +define noundef <4 x i64> @abs_i64Vec4(<4 x i64> noundef %a) #0 { +entry: +; CHECK: sub <4 x i64> zeroinitializer, %a +; CHECK: call <4 x i64> @llvm.smax.v4i64(<4 x i64> %a, <4 x i64> %{{.*}}) + %elt.abs = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a, i1 false) + ret <4 x i64> %elt.abs +} + +declare <2 x i16> @llvm.abs.v2i16(<2 x i16>, i1 immarg) +declare <3 x i32> @llvm.abs.v3i32(<3 x i32>, i1 immarg) +declare <4 x i64> @llvm.abs.v4i64(<4 x i64>, i1 immarg) diff --git a/llvm/test/CodeGen/DirectX/abs.ll b/llvm/test/CodeGen/DirectX/abs.ll new file mode 100644 index 000000000000..822580e8c089 --- /dev/null +++ b/llvm/test/CodeGen/DirectX/abs.ll @@ -0,0 +1,38 @@ +; RUN: opt -S -dxil-intrinsic-expansion < %s | FileCheck %s --check-prefixes=CHECK,EXPCHECK +; RUN: opt -S -dxil-op-lower < %s | FileCheck %s --check-prefixes=CHECK,DOPCHECK + +; Make sure dxil operation function calls for abs are generated for int16_t/int/int64_t. + +; CHECK-LABEL: abs_i16 +define noundef i16 @abs_i16(i16 noundef %a) { +entry: +; CHECK: sub i16 0, %a +; EXPCHECK: call i16 @llvm.smax.i16(i16 %a, i16 %{{.*}}) +; DOPCHECK: call i16 @dx.op.binary.i16(i32 37, i16 %a, i16 %{{.*}}) + %elt.abs = call i16 @llvm.abs.i16(i16 %a, i1 false) + ret i16 %elt.abs +} + +; CHECK-LABEL: abs_i32 +define noundef i32 @abs_i32(i32 noundef %a) { +entry: +; CHECK: sub i32 0, %a +; EXPCHECK: call i32 @llvm.smax.i32(i32 %a, i32 %{{.*}}) +; DOPCHECK: call i32 @dx.op.binary.i32(i32 37, i32 %a, i32 %{{.*}}) + %elt.abs = call i32 @llvm.abs.i32(i32 %a, i1 false) + ret i32 %elt.abs +} + +; CHECK-LABEL: abs_i64 +define noundef i64 @abs_i64(i64 noundef %a) { +entry: +; CHECK: sub i64 0, %a +; EXPCHECK: call i64 @llvm.smax.i64(i64 %a, i64 %{{.*}}) +; DOPCHECK: call i64 @dx.op.binary.i64(i32 37, i64 %a, i64 %{{.*}}) + %elt.abs = call i64 @llvm.abs.i64(i64 %a, i1 false) + ret i64 %elt.abs +} + +declare i16 @llvm.abs.i16(i16, i1 immarg) +declare i32 @llvm.abs.i32(i32, i1 immarg) +declare i64 @llvm.abs.i64(i64, i1 immarg) diff --git a/llvm/test/CodeGen/DirectX/fabs.ll b/llvm/test/CodeGen/DirectX/fabs.ll new file mode 100644 index 000000000000..3b3f8aa9a4a9 --- /dev/null +++ b/llvm/test/CodeGen/DirectX/fabs.ll @@ -0,0 +1,32 @@ +; RUN: opt -S -dxil-op-lower < %s | FileCheck %s + +; Make sure dxil operation function calls for abs are generated for float, half, and double. + + +; CHECK-LABEL: fabs_half +define noundef half @fabs_half(half noundef %a) { +entry: + ; CHECK: call half @dx.op.unary.f16(i32 6, half %{{.*}}) + %elt.abs = call half @llvm.fabs.f16(half %a) + ret half %elt.abs +} + +; CHECK-LABEL: fabs_float +define noundef float @fabs_float(float noundef %a) { +entry: +; CHECK: call float @dx.op.unary.f32(i32 6, float %{{.*}}) + %elt.abs = call float @llvm.fabs.f32(float %a) + ret float %elt.abs +} + +; CHECK-LABEL: fabs_double +define noundef double @fabs_double(double noundef %a) { +entry: +; CHECK: call double @dx.op.unary.f64(i32 6, double %{{.*}}) + %elt.abs = call double @llvm.fabs.f64(double %a) + ret double %elt.abs +} + +declare half @llvm.fabs.f16(half) +declare float @llvm.fabs.f32(float) +declare double @llvm.fabs.f64(double) -- GitLab From 79c32eb03d9ee4dd0a913c4130bc87c5e8ce7908 Mon Sep 17 00:00:00 2001 From: Farzon Lotfi <1802579+farzonl@users.noreply.github.com> Date: Fri, 22 Mar 2024 07:02:47 -0400 Subject: [PATCH 039/306] [DXIL] Add lowerings for cosine and floor (#86173) Completes #86170 Completes #86172 - `DXIL.td` - Add changes to lower the cosine and floor intrinsics to dxilOps. --- llvm/lib/Target/DirectX/DXIL.td | 6 ++++++ llvm/test/CodeGen/DirectX/cos.ll | 20 ++++++++++++++++++++ llvm/test/CodeGen/DirectX/cos_error.ll | 10 ++++++++++ llvm/test/CodeGen/DirectX/floor.ll | 20 ++++++++++++++++++++ llvm/test/CodeGen/DirectX/floor_error.ll | 10 ++++++++++ 5 files changed, 66 insertions(+) create mode 100644 llvm/test/CodeGen/DirectX/cos.ll create mode 100644 llvm/test/CodeGen/DirectX/cos_error.ll create mode 100644 llvm/test/CodeGen/DirectX/floor.ll create mode 100644 llvm/test/CodeGen/DirectX/floor_error.ll diff --git a/llvm/lib/Target/DirectX/DXIL.td b/llvm/lib/Target/DirectX/DXIL.td index ba7cd6ae9183..f7e69ebae15b 100644 --- a/llvm/lib/Target/DirectX/DXIL.td +++ b/llvm/lib/Target/DirectX/DXIL.td @@ -260,6 +260,9 @@ def Abs : DXILOpMapping<6, unary, int_fabs, def IsInf : DXILOpMapping<9, isSpecialFloat, int_dx_isinf, "Determines if the specified value is infinite.", [llvm_i1_ty, llvm_halforfloat_ty]>; +def Cos : DXILOpMapping<12, unary, int_cos, + "Returns cosine(theta) for theta in radians.", + [llvm_halforfloat_ty, LLVMMatchType<0>]>; def Sin : DXILOpMapping<13, unary, int_sin, "Returns sine(theta) for theta in radians.", [llvm_halforfloat_ty, LLVMMatchType<0>]>; @@ -279,6 +282,9 @@ def Round : DXILOpMapping<26, unary, int_round, "Returns the input rounded to the nearest integer" "within a floating-point type.", [llvm_halforfloat_ty, LLVMMatchType<0>]>; +def Floor : DXILOpMapping<27, unary, int_floor, + "Returns the largest integer that is less than or equal to the input.", + [llvm_halforfloat_ty, LLVMMatchType<0>]>; def FMax : DXILOpMapping<35, binary, int_maxnum, "Float maximum. FMax(a,b) = a > b ? a : b">; def FMin : DXILOpMapping<36, binary, int_minnum, diff --git a/llvm/test/CodeGen/DirectX/cos.ll b/llvm/test/CodeGen/DirectX/cos.ll new file mode 100644 index 000000000000..00f2e2c3f6e5 --- /dev/null +++ b/llvm/test/CodeGen/DirectX/cos.ll @@ -0,0 +1,20 @@ +; RUN: opt -S -dxil-op-lower < %s | FileCheck %s + +; Make sure dxil operation function calls for cos are generated for float and half. + +define noundef float @cos_float(float noundef %a) #0 { +entry: +; CHECK:call float @dx.op.unary.f32(i32 12, float %{{.*}}) + %elt.cos = call float @llvm.cos.f32(float %a) + ret float %elt.cos +} + +define noundef half @cos_half(half noundef %a) #0 { +entry: +; CHECK:call half @dx.op.unary.f16(i32 12, half %{{.*}}) + %elt.cos = call half @llvm.cos.f16(half %a) + ret half %elt.cos +} + +declare half @llvm.cos.f16(half) +declare float @llvm.cos.f32(float) diff --git a/llvm/test/CodeGen/DirectX/cos_error.ll b/llvm/test/CodeGen/DirectX/cos_error.ll new file mode 100644 index 000000000000..a074f5b493df --- /dev/null +++ b/llvm/test/CodeGen/DirectX/cos_error.ll @@ -0,0 +1,10 @@ +; RUN: not opt -S -dxil-op-lower %s 2>&1 | FileCheck %s + +; DXIL operation cos does not support double overload type +; CHECK: LLVM ERROR: Invalid Overload Type + +define noundef double @cos_double(double noundef %a) { +entry: + %elt.cos = call double @llvm.cos.f64(double %a) + ret double %elt.cos +} diff --git a/llvm/test/CodeGen/DirectX/floor.ll b/llvm/test/CodeGen/DirectX/floor.ll new file mode 100644 index 000000000000..b033e2eaa491 --- /dev/null +++ b/llvm/test/CodeGen/DirectX/floor.ll @@ -0,0 +1,20 @@ +; RUN: opt -S -dxil-op-lower < %s | FileCheck %s + +; Make sure dxil operation function calls for floor are generated for float and half. + +define noundef float @floor_float(float noundef %a) #0 { +entry: +; CHECK:call float @dx.op.unary.f32(i32 27, float %{{.*}}) + %elt.floor = call float @llvm.floor.f32(float %a) + ret float %elt.floor +} + +define noundef half @floor_half(half noundef %a) #0 { +entry: +; CHECK:call half @dx.op.unary.f16(i32 27, half %{{.*}}) + %elt.floor = call half @llvm.floor.f16(half %a) + ret half %elt.floor +} + +declare half @llvm.floor.f16(half) +declare float @llvm.floor.f32(float) diff --git a/llvm/test/CodeGen/DirectX/floor_error.ll b/llvm/test/CodeGen/DirectX/floor_error.ll new file mode 100644 index 000000000000..3b51a4b543b7 --- /dev/null +++ b/llvm/test/CodeGen/DirectX/floor_error.ll @@ -0,0 +1,10 @@ +; RUN: not opt -S -dxil-op-lower %s 2>&1 | FileCheck %s + +; DXIL operation floor does not support double overload type +; CHECK: LLVM ERROR: Invalid Overload Type + +define noundef double @floor_double(double noundef %a) { +entry: + %elt.floor = call double @llvm.floor.f64(double %a) + ret double %elt.floor +} -- GitLab From db33444ffa7e210e7040d8def958a14171f52eef Mon Sep 17 00:00:00 2001 From: "Oleksandr \"Alex\" Zinenko" Date: Fri, 22 Mar 2024 12:03:28 +0100 Subject: [PATCH 040/306] CODEOWNERS: extend scope of MLIR transform dialect There are a bunch of related directories in another dialects. --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index fea132c8fe78..77ba81c58c5d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -96,6 +96,7 @@ clang/test/AST/Interp/ @tbaederr # Transform Dialect in MLIR. /mlir/include/mlir/Dialect/Transform/* @ftynse @nicolasvasilache /mlir/lib/Dialect/Transform/* @ftynse @nicolasvasilache +/mlir/**/*TransformOps* @ftynse @nicolasvasilache # SPIR-V Dialect in MLIR. /mlir/**/SPIRV/ @antiagainst @kuhar -- GitLab From e925968e7815ac3810fdb54bb884b8a8bed02eb5 Mon Sep 17 00:00:00 2001 From: Balazs Benics Date: Fri, 22 Mar 2024 12:04:44 +0100 Subject: [PATCH 041/306] [analyzer] Support C++23 static operator calls (#84972) Made by following: https://github.com/llvm/llvm-project/pull/83585#issuecomment-1980340866 Thanks for the details Tomek! CPP-5080 --- clang/docs/ReleaseNotes.rst | 1 + .../Core/PathSensitive/CallEvent.h | 72 +++++++++++++++++++ clang/lib/StaticAnalyzer/Core/CallEvent.cpp | 5 +- .../Core/ExprEngineCallAndReturn.cpp | 1 + clang/test/Analysis/cxx23-static-operator.cpp | 38 ++++++++++ 5 files changed, 116 insertions(+), 1 deletion(-) create mode 100644 clang/test/Analysis/cxx23-static-operator.cpp diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index fd12bb41be47..45b2e01af997 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -576,6 +576,7 @@ Static Analyzer - Fixed crashing on loops if the loop variable was declared in switch blocks but not under any case blocks if ``unroll-loops=true`` analyzer config is set. (#GH68819) +- Support C++23 static operator calls. (#GH84972) New features ^^^^^^^^^^^^ diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h index 0d36587484bf..549c864dc91e 100644 --- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h +++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h @@ -59,6 +59,7 @@ namespace ento { enum CallEventKind { CE_Function, + CE_CXXStaticOperator, CE_CXXMember, CE_CXXMemberOperator, CE_CXXDestructor, @@ -709,6 +710,77 @@ public: } }; +/// Represents a static C++ operator call. +/// +/// "A" in this example. +/// However, "B" and "C" are represented by SimpleFunctionCall. +/// \code +/// struct S { +/// int pad; +/// static void operator()(int x, int y); +/// }; +/// S s{10}; +/// void (*fptr)(int, int) = &S::operator(); +/// +/// s(1, 2); // A +/// S::operator()(1, 2); // B +/// fptr(1, 2); // C +/// \endcode +class CXXStaticOperatorCall : public SimpleFunctionCall { + friend class CallEventManager; + +protected: + CXXStaticOperatorCall(const CXXOperatorCallExpr *CE, ProgramStateRef St, + const LocationContext *LCtx, + CFGBlock::ConstCFGElementRef ElemRef) + : SimpleFunctionCall(CE, St, LCtx, ElemRef) {} + CXXStaticOperatorCall(const CXXStaticOperatorCall &Other) = default; + + void cloneTo(void *Dest) const override { + new (Dest) CXXStaticOperatorCall(*this); + } + +public: + const CXXOperatorCallExpr *getOriginExpr() const override { + return cast(SimpleFunctionCall::getOriginExpr()); + } + + unsigned getNumArgs() const override { + // Ignore the object parameter that is not used for static member functions. + assert(getOriginExpr()->getNumArgs() > 0); + return getOriginExpr()->getNumArgs() - 1; + } + + const Expr *getArgExpr(unsigned Index) const override { + // Ignore the object parameter that is not used for static member functions. + return getOriginExpr()->getArg(Index + 1); + } + + std::optional + getAdjustedParameterIndex(unsigned ASTArgumentIndex) const override { + // Ignore the object parameter that is not used for static member functions. + if (ASTArgumentIndex == 0) + return std::nullopt; + return ASTArgumentIndex - 1; + } + + unsigned getASTArgumentIndex(unsigned CallArgumentIndex) const override { + // Account for the object parameter for the static member function. + return CallArgumentIndex + 1; + } + + OverloadedOperatorKind getOverloadedOperator() const { + return getOriginExpr()->getOperator(); + } + + Kind getKind() const override { return CE_CXXStaticOperator; } + StringRef getKindAsString() const override { return "CXXStaticOperatorCall"; } + + static bool classof(const CallEvent *CA) { + return CA->getKind() == CE_CXXStaticOperator; + } +}; + /// Represents a non-static C++ member function call. /// /// Example: \c obj.fun() diff --git a/clang/lib/StaticAnalyzer/Core/CallEvent.cpp b/clang/lib/StaticAnalyzer/Core/CallEvent.cpp index bc14aea27f67..0e317ec765ec 100644 --- a/clang/lib/StaticAnalyzer/Core/CallEvent.cpp +++ b/clang/lib/StaticAnalyzer/Core/CallEvent.cpp @@ -1408,9 +1408,12 @@ CallEventManager::getSimpleCall(const CallExpr *CE, ProgramStateRef State, if (const auto *OpCE = dyn_cast(CE)) { const FunctionDecl *DirectCallee = OpCE->getDirectCallee(); - if (const auto *MD = dyn_cast(DirectCallee)) + if (const auto *MD = dyn_cast(DirectCallee)) { if (MD->isImplicitObjectMemberFunction()) return create(OpCE, State, LCtx, ElemRef); + if (MD->isStatic()) + return create(OpCE, State, LCtx, ElemRef); + } } else if (CE->getCallee()->getType()->isBlockPointerType()) { return create(CE, State, LCtx, ElemRef); diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngineCallAndReturn.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngineCallAndReturn.cpp index 4755b6bfa6dc..9d3e4fc944fb 100644 --- a/clang/lib/StaticAnalyzer/Core/ExprEngineCallAndReturn.cpp +++ b/clang/lib/StaticAnalyzer/Core/ExprEngineCallAndReturn.cpp @@ -846,6 +846,7 @@ ExprEngine::mayInlineCallKind(const CallEvent &Call, const ExplodedNode *Pred, const StackFrameContext *CallerSFC = CurLC->getStackFrame(); switch (Call.getKind()) { case CE_Function: + case CE_CXXStaticOperator: case CE_Block: break; case CE_CXXMember: diff --git a/clang/test/Analysis/cxx23-static-operator.cpp b/clang/test/Analysis/cxx23-static-operator.cpp new file mode 100644 index 000000000000..f380bd0dfa42 --- /dev/null +++ b/clang/test/Analysis/cxx23-static-operator.cpp @@ -0,0 +1,38 @@ +// RUN: %clang_analyze_cc1 -std=c++2b -verify %s \ +// RUN: -analyzer-checker=core,debug.ExprInspection + +template void clang_analyzer_dump(T); + +struct Adder { + int data; + static int operator()(int x, int y) { + clang_analyzer_dump(x); // expected-warning {{1}} + clang_analyzer_dump(y); // expected-warning {{2}} + return x + y; + } +}; + +void static_operator_call_inlines() { + Adder s{10}; + clang_analyzer_dump(s(1, 2)); // expected-warning {{3}} +} + +struct DataWithCtor { + int x; + int y; + DataWithCtor(int parm) : x(parm + 10), y(parm + 20) { + clang_analyzer_dump(this); // expected-warning {{&v}} + } +}; + +struct StaticSubscript { + static void operator[](DataWithCtor v) { + clang_analyzer_dump(v.x); // expected-warning {{20}} + clang_analyzer_dump(v.y); // expected-warning {{30}} + } +}; + +void top() { + StaticSubscript s; + s[DataWithCtor{10}]; +} -- GitLab From 74c3150ffc86a149abc68acdf8af1eed1ea0f038 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Fri, 22 Mar 2024 11:23:25 +0000 Subject: [PATCH 042/306] [X86] Add shuffle tests from Issue #86076 SLP should be doing a better job, but both shuffles lower to poorer codegen than necessary --- .../CodeGen/X86/vector-shuffle-512-v16.ll | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/llvm/test/CodeGen/X86/vector-shuffle-512-v16.ll b/llvm/test/CodeGen/X86/vector-shuffle-512-v16.ll index dfa7f2dbdaee..c981d973fef3 100644 --- a/llvm/test/CodeGen/X86/vector-shuffle-512-v16.ll +++ b/llvm/test/CodeGen/X86/vector-shuffle-512-v16.ll @@ -177,6 +177,36 @@ define <16 x float> @shuffle_v16f32_02_03_16_17_06_07_20_21_10_11_24_25_14_15_28 ret <16 x float> %shuffle } +; PR86076 +define <16 x float> @shuffle_f32_v16f32_00_08_01_09_02_10_03_11_04_12_05_13_06_14_07_15(float %a0, float %a1) { +; ALL-LABEL: shuffle_f32_v16f32_00_08_01_09_02_10_03_11_04_12_05_13_06_14_07_15: +; ALL: # %bb.0: +; ALL-NEXT: vbroadcastss %xmm0, %ymm0 +; ALL-NEXT: vbroadcastss %xmm1, %ymm1 +; ALL-NEXT: vunpcklps {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] +; ALL-NEXT: vinsertf64x4 $1, %ymm0, %zmm0, %zmm0 +; ALL-NEXT: retq + %v0 = insertelement <8 x float> poison, float %a0, i64 0 + %v1 = insertelement <8 x float> poison, float %a1, i64 0 + %b0 = shufflevector <8 x float> %v0, <8 x float> poison, <8 x i32> zeroinitializer + %b1 = shufflevector <8 x float> %v1, <8 x float> poison, <8 x i32> zeroinitializer + %r = shufflevector <8 x float> %b0, <8 x float> %b1, <16 x i32> + ret <16 x float> %r +} + +; PR86076 +define <16 x float> @shuffle_f32_v16f32_00_08_00_08_00_08_00_08_00_08_00_08_00_08_00_08(float %a0, float %a1) { +; ALL-LABEL: shuffle_f32_v16f32_00_08_00_08_00_08_00_08_00_08_00_08_00_08_00_08: +; ALL: # %bb.0: +; ALL-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],zero,zero +; ALL-NEXT: vbroadcastsd %xmm0, %zmm0 +; ALL-NEXT: retq + %v0 = insertelement <8 x float> poison, float %a0, i64 0 + %v1 = insertelement <8 x float> poison, float %a1, i64 0 + %sv = shufflevector <8 x float> %v0, <8 x float> %v1, <16 x i32> + ret <16 x float> %sv +} + define <16 x i32> @shuffle_v16i32_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00(<16 x i32> %a, <16 x i32> %b) { ; ALL-LABEL: shuffle_v16i32_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00: ; ALL: # %bb.0: -- GitLab From ceabaa7e7a2d02b20cbd2b31e8336dedb1d4d9f5 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Fri, 22 Mar 2024 11:48:03 +0000 Subject: [PATCH 043/306] [DAG] Fix some missing formatting when I rewrote the SUB(MAX,MIN) -> ABD patterns. NFC. --- llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index db81f9199170..dcd0310734ad 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -4024,13 +4024,13 @@ SDValue DAGCombiner::visitSUB(SDNode *N) { } // smax(a,b) - smin(a,b) --> abds(a,b) - if (hasOperation(ISD::ABDS, VT) && + if (hasOperation(ISD::ABDS, VT) && sd_match(N0, m_SMax(m_Value(A), m_Value(B))) && sd_match(N1, m_SMin(m_Specific(A), m_Specific(B)))) return DAG.getNode(ISD::ABDS, DL, VT, A, B); // umax(a,b) - umin(a,b) --> abdu(a,b) - if (hasOperation(ISD::ABDU, VT) && + if (hasOperation(ISD::ABDU, VT) && sd_match(N0, m_UMax(m_Value(A), m_Value(B))) && sd_match(N1, m_UMin(m_Specific(A), m_Specific(B)))) return DAG.getNode(ISD::ABDU, DL, VT, A, B); -- GitLab From c41286af3f30e099556c6edbef0001466afaefcb Mon Sep 17 00:00:00 2001 From: Pablo Antonio Martinez Date: Fri, 22 Mar 2024 11:53:29 +0000 Subject: [PATCH 044/306] [mlir][linalg] Emit a warning when tile_using_forall generates non thread-safe code (#80813) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Description** The documentation of `transform.structured.tile_using_forall` says: _"It is the user’s responsibility to ensure that num_threads/tile_sizes is a valid tiling specification (i.e. that only tiles parallel dimensions, e.g. in the Linalg case)."_ In other words, tiling a non-parallel dimension would generate code with data races which is not safe to parallelize. For example, consider this example (included in the tests in this PR): ``` func.func @tile_thread_safety2(%arg0: tensor<100x300x8xf32>, %arg1: tensor<300x8xf32>) -> tensor<300x8xf32> { %0 = scf.forall (%arg2) in (8) shared_outs(%arg3 = %arg1) -> (tensor<300x8xf32>) { %1 = affine.min #map(%arg2) %2 = affine.max #map1(%1) %3 = affine.apply #map2(%arg2) %extracted_slice = tensor.extract_slice %arg0[%3, 0, 0] [%2, 300, 8] [1, 1, 1] : tensor<100x300x8xf32> to tensor %4 = linalg.generic {indexing_maps = [#map3, #map4], iterator_types = ["reduction", "parallel", "parallel"]} ins(%extracted_slice : tensor) outs(%arg3 : tensor<300x8xf32>) { ^bb0(%in: f32, %out: f32): %5 = arith.addf %in, %out : f32 linalg.yield %5 : f32 } -> tensor<300x8xf32> scf.forall.in_parallel { tensor.parallel_insert_slice %4 into %arg3[0, 0] [300, 8] [1, 1] : tensor<300x8xf32> into tensor<300x8xf32> } } return %0 : tensor<300x8xf32> } ``` We can easily see that this is not safe to parallelize because all threads would be writing to the same position in `%arg3` (in the `scf.forall.in_parallel`. This PR detects wether it's safe to `tile_using_forall` and emits a warning in the case it is not. **Brief explanation** It first generates a vector of affine expressions representing the tile values and stores it in `dimExprs`. These affine expressions are compared with the affine expressions coming from the results of the affine map of each output in the linalg op. So going back to the previous example, the original transform is: ``` #map = affine_map<(d0, d1, d2) -> (d0, d1, d2)> #map1 = affine_map<(d0, d1, d2) -> (d1, d2)> func.func @tile_thread_safety2(%arg0: tensor<100x300x8xf32>, %arg1: tensor<300x8xf32>) -> tensor<300x8xf32> { // expected-warning@+1 {{tiling is not thread safe at axis #0}} %0 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["reduction", "parallel", "parallel"]} ins(%arg0 : tensor<100x300x8xf32>) outs(%arg1 : tensor<300x8xf32>) { ^bb0(%in: f32, %out: f32): %1 = arith.addf %in, %out : f32 linalg.yield %1 : f32 } -> tensor<300x8xf32> return %0 : tensor<300x8xf32> } module attributes {transform.with_named_sequence} { transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) { %0 = transform.structured.match ops{["linalg.generic"]} in %arg0 : (!transform.any_op) -> !transform.any_op %forall, %tiled_generic = transform.structured.tile_using_forall %0 num_threads [8] : (!transform.any_op) -> (!transform.any_op, !transform.any_op) transform.yield } } ``` The `num_threads` attribute would be represented as `(d0)`. Because the linalg op has only one output (`arg1`) it would only check against the results of `#map1`, which are `(d1, d2)`. The idea is to check that all affine expressions in `dimExprs` are present in the output affine map. In this example, `d0` is not in `(d1, d2)`, so tiling that axis is considered not thread safe. --- .../Linalg/TransformOps/LinalgTransformOps.td | 4 +- mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp | 38 ++++- mlir/test/Dialect/Linalg/tile-to-forall.mlir | 141 ++++++++++++++++++ 3 files changed, 180 insertions(+), 3 deletions(-) diff --git a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td index 4f34016066b4..c260fe3f7a46 100644 --- a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td +++ b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td @@ -1918,7 +1918,9 @@ def TileUsingForallOp : It is the user's responsibility to ensure that `num_threads/tile_sizes` is a valid tiling specification (i.e. that only tiles parallel dimensions, - e.g. in the Linalg case). + e.g. in the Linalg case). If the dimension is not parallelizable, a warning + is issued to notify the user that the generated code is not safe to + parallelize. If non-empty, the `mapping` is added as an attribute to the resulting `scf.forall`. diff --git a/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp b/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp index 30aed850bed8..462f692615fa 100644 --- a/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp +++ b/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp @@ -304,6 +304,28 @@ static void calculateTileOffsetsAndSizes( } } +/// Returns a vector of bools representing if, for each axis, `op` can be tiled +/// without incurring in a race condition and thus it is thread-safe to do the +/// tiling. This is checked by iterating over numThreads and ensuring that the +/// corresponding iterator type is "parallel". If it is not, then we know that +/// such dimension is unsafe to tile. +SmallVector safeToTileToForall(mlir::MLIRContext *ctx, LinalgOp linalgOp, + ArrayRef numThreads) { + auto iterators = linalgOp.getIteratorTypesArray(); + SmallVector safeToTile(numThreads.size(), true); + + for (unsigned i = 0, e = numThreads.size(); i != e; i++) { + if (auto attr = llvm::dyn_cast_if_present(numThreads[i])) { + if (cast(attr).getValue().getSExtValue() > 1) { + safeToTile[i] = iterators[i] == utils::IteratorType::parallel; + } + } else { + safeToTile[i] = iterators[i] == utils::IteratorType::parallel; + } + } + return safeToTile; +} + /// Rewrite a TilingInterface `op` to a tiled `scf.forall`. The /// tiling is specified by the number of tiles/threads `numThreads` and the /// optional nominal tile size `nominalTileSizes`. If `nominalTilSizes` is @@ -314,8 +336,10 @@ static void calculateTileOffsetsAndSizes( /// size of data. /// It is the user's responsibility to ensure that `numThreads` is a valid /// tiling specification (i.e. that only tiles parallel dimensions, e.g. in the -/// Linalg case). If `omitTileOffsetBoundsCheck` is true, then the function will -/// assume that `tileSize[i] * (numThread[i] -1) <= dimSize[i]` holds. +/// Linalg case). If the dimension is not parallelizable, a warning is issued to +/// notify the user that the generated code is not safe to parallelize. If +/// `omitTileOffsetBoundsCheck` is true, then the function will assume that +/// `tileSize[i] * (numThread[i] -1) <= dimSize[i]` holds. static FailureOr tileToForallOpImpl( RewriterBase &b, TilingInterface op, ArrayRef numThreads, std::optional> nominalTileSizes, @@ -344,6 +368,16 @@ static FailureOr tileToForallOpImpl( return getValueOrCreateConstantIndexOp(b, loc, ofr); })); + LinalgOp linalgOp = dyn_cast(op.getOperation()); + if (linalgOp) { + // Check if tiling is thread safe and print a warning if not. + SmallVector tilingSafety = + safeToTileToForall(b.getContext(), linalgOp, numThreads); + for (size_t i = 0; i < tilingSafety.size(); i++) + if (!tilingSafety[i]) + op.emitWarning() << "tiling is not thread safe at axis #" << i; + } + // 1. Create the ForallOp. We don't use the lambda body-builder // version because we require the use of RewriterBase in the body, so we // manually move the insertion point to the body below. diff --git a/mlir/test/Dialect/Linalg/tile-to-forall.mlir b/mlir/test/Dialect/Linalg/tile-to-forall.mlir index abd807b3e4d3..12e2dea5530b 100644 --- a/mlir/test/Dialect/Linalg/tile-to-forall.mlir +++ b/mlir/test/Dialect/Linalg/tile-to-forall.mlir @@ -586,3 +586,144 @@ module attributes {transform.with_named_sequence} { transform.yield } } + +// ----- + +#map = affine_map<(d0, d1) -> (d0, d1)> +#map1 = affine_map<(d0, d1) -> (d0)> + +func.func @tile_thread_safety1(%arg0: tensor<100x300xf32>, %arg1: tensor<100xf32>) -> tensor<100xf32> { + // expected-warning@below {{tiling is not thread safe at axis #1}} + %0 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "reduction"]} ins(%arg0 : tensor<100x300xf32>) outs(%arg1 : tensor<100xf32>) { + ^bb0(%in: f32, %out: f32): + %1 = arith.addf %in, %out : f32 + linalg.yield %1 : f32 + } -> tensor<100xf32> + return %0 : tensor<100xf32> +} + +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) { + %0 = transform.structured.match ops{["linalg.generic"]} in %arg0 : (!transform.any_op) -> !transform.any_op + %forall, %tiled_generic = transform.structured.tile_using_forall %0 num_threads [4, 2] + : (!transform.any_op) -> (!transform.any_op, !transform.any_op) + transform.yield + } +} + +// ----- + +#map = affine_map<(d0, d1, d2) -> (d0, d1, d2)> +#map1 = affine_map<(d0, d1, d2) -> (d1, d2)> + +func.func @tile_thread_safety2(%arg0: tensor<100x300x8xf32>, %arg1: tensor<300x8xf32>) -> tensor<300x8xf32> { + // expected-warning@below {{tiling is not thread safe at axis #0}} + %0 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["reduction", "parallel", "parallel"]} ins(%arg0 : tensor<100x300x8xf32>) outs(%arg1 : tensor<300x8xf32>) { + ^bb0(%in: f32, %out: f32): + %1 = arith.addf %in, %out : f32 + linalg.yield %1 : f32 + } -> tensor<300x8xf32> + return %0 : tensor<300x8xf32> +} + +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) { + %0 = transform.structured.match ops{["linalg.generic"]} in %arg0 : (!transform.any_op) -> !transform.any_op + %forall, %tiled_generic = transform.structured.tile_using_forall %0 num_threads [8] + : (!transform.any_op) -> (!transform.any_op, !transform.any_op) + transform.yield + } +} + +// ----- + +#map = affine_map<(d0, d1, d2) -> (d0, d1, d2)> +#map1 = affine_map<(d0, d1, d2) -> (d0, d2)> + +func.func @tile_thread_safety3(%arg0: tensor<100x300x8xf32>, %arg1: tensor<100x8xf32>) -> tensor<100x8xf32> { + // expected-warning@below {{tiling is not thread safe at axis #1}} + %0 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "reduction", "parallel"]} ins(%arg0 : tensor<100x300x8xf32>) outs(%arg1 : tensor<100x8xf32>) { + ^bb0(%in: f32, %out: f32): + %1 = arith.addf %in, %out : f32 + linalg.yield %1 : f32 + } -> tensor<100x8xf32> + return %0 : tensor<100x8xf32> +} + +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) { + %0 = transform.structured.match ops{["linalg.generic"]} in %arg0 : (!transform.any_op) -> !transform.any_op + %forall, %tiled_generic = transform.structured.tile_using_forall %0 num_threads [8, 4, 2] + : (!transform.any_op) -> (!transform.any_op, !transform.any_op) + transform.yield + } +} + +// ----- + +#map = affine_map<(d0, d1, d2) -> (d0, d1, d2)> +#map1 = affine_map<(d0, d1, d2) -> (d0, d2)> +#map2 = affine_map<(d0, d1, d2) -> (d2)> + +func.func @tile_thread_safety4(%arg0: tensor<100x300x8xf32>, %arg1: tensor<100x8xf32>, %arg2 : tensor<8xf32>) -> (tensor<100x8xf32>, tensor<8xf32>) { + // expected-warning@+2 {{tiling is not thread safe at axis #0}} + // expected-warning@below {{tiling is not thread safe at axis #1}} + %0:2 = linalg.generic {indexing_maps = [#map, #map1, #map2], iterator_types = ["reduction", "reduction", "parallel"]} ins(%arg0 : tensor<100x300x8xf32>) outs(%arg1, %arg2 : tensor<100x8xf32>, tensor<8xf32>) { + ^bb0(%in: f32, %out1: f32, %out2: f32): + %1 = arith.addf %in, %out1 : f32 + %2 = arith.addf %in, %out2 : f32 + linalg.yield %1, %2 : f32, f32 + } -> (tensor<100x8xf32>, tensor<8xf32>) + return %0#0, %0#1 : tensor<100x8xf32>, tensor<8xf32> +} + +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) { + %0 = transform.structured.match ops{["linalg.generic"]} in %arg0 : (!transform.any_op) -> !transform.any_op + %forall, %tiled_generic = transform.structured.tile_using_forall %0 num_threads [8, 4, 2] + : (!transform.any_op) -> (!transform.any_op, !transform.any_op) + transform.yield + } +} + +// ----- + +#map = affine_map<(d0, d1) -> (d0, d1)> +#map1 = affine_map<(d0, d1) -> (d0)> + +func.func @tile_thread_safety5(%arg0: tensor<100x300xf32>, %arg1: tensor<100xf32>) -> tensor<100xf32> { + // expected-warning@below {{tiling is not thread safe at axis #1}} + %0 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "reduction"]} ins(%arg0 : tensor<100x300xf32>) outs(%arg1 : tensor<100xf32>) { + ^bb0(%in: f32, %out: f32): + %1 = arith.addf %in, %out : f32 + linalg.yield %1 : f32 + } -> tensor<100xf32> + return %0 : tensor<100xf32> +} + +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) { + %0 = transform.structured.match ops{["linalg.generic"]} in %arg0 : (!transform.any_op) -> !transform.any_op + %forall, %tiled_generic = transform.structured.tile_using_forall %0 tile_sizes [10, 1] + : (!transform.any_op) -> (!transform.any_op, !transform.any_op) + transform.yield + } +} + +// ----- + +func.func @tile_thread_safety6(%A: tensor, %B: tensor, %C: tensor) -> tensor { + // expected-warning@below {{tiling is not thread safe at axis #2}} + %0 = linalg.matmul ins(%A, %B : tensor, tensor) + outs(%C : tensor) -> (tensor) + return %0 : tensor +} + +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) { + %0 = transform.structured.match ops{["linalg.matmul"]} in %arg0 : (!transform.any_op) -> !transform.any_op + %forall, %tiled_generic = transform.structured.tile_using_forall %0 num_threads [2, 0, 8] + : (!transform.any_op) -> (!transform.any_op, !transform.any_op) + transform.yield + } +} -- GitLab From d03f470cbdbae3f86469ea4d79bb54d3ef680512 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Fri, 22 Mar 2024 06:54:33 -0500 Subject: [PATCH 045/306] [Clang] Make '-frtlib-add-rpath' include the standard library directory (#86217) Summary: The original intention of the `openmp-add-rpath` option was to add the rpath to the language runtime directory. However, the current implementation only adds it to the compiler's resource directory. This patch adds support for appending the `-rpath` to the compiler's standard library directory as well. Currently this is `/../lib/`. --- clang/lib/Driver/ToolChains/CommonArgs.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp index 447886531363..6b1fbba7abd0 100644 --- a/clang/lib/Driver/ToolChains/CommonArgs.cpp +++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp @@ -1142,7 +1142,11 @@ void tools::addArchSpecificRPath(const ToolChain &TC, const ArgList &Args, options::OPT_fno_rtlib_add_rpath, false)) return; - for (const auto &CandidateRPath : TC.getArchSpecificLibPaths()) { + SmallVector CandidateRPaths(TC.getArchSpecificLibPaths()); + if (const auto CandidateRPath = TC.getStdlibPath()) + CandidateRPaths.emplace_back(*CandidateRPath); + + for (const auto &CandidateRPath : CandidateRPaths) { if (TC.getVFS().exists(CandidateRPath)) { CmdArgs.push_back("-rpath"); CmdArgs.push_back(Args.MakeArgString(CandidateRPath)); -- GitLab From d51f1c442b1dc999267726a33e25b7d019726c89 Mon Sep 17 00:00:00 2001 From: Farzon Lotfi <1802579+farzonl@users.noreply.github.com> Date: Fri, 22 Mar 2024 08:14:26 -0400 Subject: [PATCH 046/306] [DirectX][Docs] Add DXILIntrinsicExpansion Pass to DXILArchitecture.rst (#86198) Completes #84839 --------- Co-authored-by: Farzon Lotfi --- llvm/docs/DirectX/DXILArchitecture.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/llvm/docs/DirectX/DXILArchitecture.rst b/llvm/docs/DirectX/DXILArchitecture.rst index d6712bea4f77..32b1e72deae7 100644 --- a/llvm/docs/DirectX/DXILArchitecture.rst +++ b/llvm/docs/DirectX/DXILArchitecture.rst @@ -61,6 +61,19 @@ on the utilities described in "Common Code" above in order to share logic with both the DirectX backend and with Clang's codegen of HLSL support as much as possible. +The DirectX Intrinsic Expansion Pass +==================================== +There are intrinsics that don't map directly to DXIL Ops. In some cases +an intrinsic needs to be expanded to a set of LLVM IR instructions. In +other cases an intrinsic needs modifications to the arguments or return +values of a DXIL Op. The `DXILIntrinsicExpansion` pass handles all +the cases where our intrinsics don't have a one to one mapping. This +pass may also be used when the expansion is specific to DXIL to keep +implementation details out of CodeGen. Finally, there is an expectation +that we maintain vector types through this pass. Therefore, best +practice would be to avoid scalarization in this pass. + + The DirectX Backend =================== -- GitLab From fe64b26df9429c82f706424dcdae3d65723c3e5e Mon Sep 17 00:00:00 2001 From: Orlando Cazalet-Hyams Date: Fri, 22 Mar 2024 12:20:52 +0000 Subject: [PATCH 047/306] NFC Rename LoadBitcodeIntoNewDbgInforFormat to LoadBitcodeIntoNewDbgInfoFormat (drop additional 'r' before Format) --- llvm/lib/Bitcode/Reader/BitcodeReader.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp index 3fc8141381c6..6ee93f17792b 100644 --- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp +++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp @@ -104,7 +104,7 @@ static cl::opt ExpandConstantExprs( /// of debug intrinsics). UNSET is treated as FALSE, so the default action /// is to do nothing. Individual tools can override this to incrementally add /// support for the RemoveDIs format. -cl::opt LoadBitcodeIntoNewDbgInforFormat( +cl::opt LoadBitcodeIntoNewDbgInfoFormat( "load-bitcode-into-experimental-debuginfo-iterators", cl::Hidden, cl::desc("Load bitcode directly into the new debug info format (regardless " "of input format)")); @@ -4300,11 +4300,11 @@ Error BitcodeReader::parseGlobalIndirectSymbolRecord( Error BitcodeReader::parseModule(uint64_t ResumeBit, bool ShouldLazyLoadMetadata, ParserCallbacks Callbacks) { - // Load directly into RemoveDIs format if LoadBitcodeIntoNewDbgInforFormat + // Load directly into RemoveDIs format if LoadBitcodeIntoNewDbgInfoFormat // has been set to true (default action: load into the old debug format). TheModule->IsNewDbgInfoFormat = UseNewDbgInfoFormat && - LoadBitcodeIntoNewDbgInforFormat == cl::boolOrDefault::BOU_TRUE; + LoadBitcodeIntoNewDbgInfoFormat == cl::boolOrDefault::BOU_TRUE; this->ValueTypeCallback = std::move(Callbacks.ValueType); if (ResumeBit) { -- GitLab From e1f50fdc03efecb5da39c1df4fc08d2ce5da90e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ingo=20M=C3=BCller?= Date: Fri, 22 Mar 2024 13:35:36 +0100 Subject: [PATCH 048/306] [mlir] Remove unused and untested `shouldSplitInputFile`. (#85622) This was changed by #84765 but turned out to be buggy. Since it isn't used and isn't tested, it is probably best to remove it. --- mlir/include/mlir/Tools/mlir-opt/MlirOptMain.h | 1 - 1 file changed, 1 deletion(-) diff --git a/mlir/include/mlir/Tools/mlir-opt/MlirOptMain.h b/mlir/include/mlir/Tools/mlir-opt/MlirOptMain.h index 8adc80908de1..4f7f83cdb473 100644 --- a/mlir/include/mlir/Tools/mlir-opt/MlirOptMain.h +++ b/mlir/include/mlir/Tools/mlir-opt/MlirOptMain.h @@ -144,7 +144,6 @@ public: splitInputFileFlag = std::move(splitMarker); return *this; } - bool shouldSplitInputFile() const { return splitInputFileFlag.empty(); } StringRef inputSplitMarker() const { return splitInputFileFlag; } /// Set whether to merge the output chunks into one file using the given -- GitLab From 83da7b6338053ca04cf0afe3c70ef5b8a9f6d300 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ingo=20M=C3=BCller?= Date: Fri, 22 Mar 2024 13:36:07 +0100 Subject: [PATCH 049/306] [mlir] Extend split marker tests of `mlir-opt` and `mlir-pdll`. (#85620) Recently #84765 made the split markers of various tools configurable but did not test *not* using the split markers for two of them. This PR adds those tests. --- mlir/test/mlir-opt/split-markers.mlir | 32 ++++++++++++++++---------- mlir/test/mlir-pdll/split-markers.pdll | 12 ++++++++++ 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/mlir/test/mlir-opt/split-markers.mlir b/mlir/test/mlir-opt/split-markers.mlir index 665a37f31770..f372654bcc8d 100644 --- a/mlir/test/mlir-opt/split-markers.mlir +++ b/mlir/test/mlir-opt/split-markers.mlir @@ -1,12 +1,17 @@ // Check near-miss mechanics: // RUN: mlir-opt --split-input-file --verify-diagnostics %s 2> %t \ -// RUN: && FileCheck --input-file %t %s +// RUN: && FileCheck --input-file %t --check-prefix=CHECK-DEFAULT %s // RUN: cat %t // Check that (1) custom input splitter and (2) custom output splitters work. -// RUN: mlir-opt %s -split-input-file="// CHECK: ""----" \ +// RUN: mlir-opt %s -split-input-file="// CHECK-DEFAULT: ""----" \ // RUN: -output-split-marker="// ---- next split ----" \ -// RUN: | FileCheck --check-prefix=CHECK-SPLITTERS %s +// RUN: | FileCheck --check-prefix=CHECK-CUSTOM %s + +// Check that (3) the input is not split if `-split-input-file` is not given. +// RUN: mlir-opt %s 2> %t \ +// RUN: || FileCheck --input-file %t --check-prefix=CHECK-NOSPLIT %s +// RUN: cat %t func.func @main() {return} @@ -14,22 +19,25 @@ func.func @main() {return} // expected-note @+1 {{see existing symbol definition here}} func.func @foo() { return } -// CHECK: warning: near miss with file split marker -// CHECK: ---- +// CHECK-DEFAULT: warning: near miss with file split marker +// CHECK-DEFAULT: ---- // ---- +// CHECK-NOSPLIT: error: redefinition of symbol named 'main' +func.func @main() {return} + // expected-error @+1 {{redefinition of symbol named 'foo'}} func.func @foo() { return } -// CHECK: warning: near miss with file split marker -// CHECK: ---- +// CHECK-DEFAULT: warning: near miss with file split marker +// CHECK-DEFAULT: ---- // ---- func.func @bar2() {return } // No error flagged at the end for a near miss. // ---- -// CHECK-SPLITTERS: module -// CHECK-SPLITTERS: ---- next split ---- -// CHECK-SPLITTERS: module -// CHECK-SPLITTERS: ---- next split ---- -// CHECK-SPLITTERS: module +// CHECK-CUSTOM: module +// CHECK-CUSTOM: ---- next split ---- +// CHECK-CUSTOM: module +// CHECK-CUSTOM: ---- next split ---- +// CHECK-CUSTOM: module diff --git a/mlir/test/mlir-pdll/split-markers.pdll b/mlir/test/mlir-pdll/split-markers.pdll index 45e409a83836..2b314538004f 100644 --- a/mlir/test/mlir-pdll/split-markers.pdll +++ b/mlir/test/mlir-pdll/split-markers.pdll @@ -9,6 +9,10 @@ // RUN: -split-input-file="// ""=====" -output-split-marker "// #####" \ // RUN: | FileCheck -check-prefix=CHECK-CUSTOM %s +// Check that (5) the input is not split if `-split-input-file` is not given. +// RUN: mlir-pdll %s \ +// RUN: | FileCheck -check-prefix=CHECK-NOSPLIT %s + // CHECK-DEFAULT: Module // CHECK-DEFAULT-NEXT: PatternDecl // CHECK-DEFAULT-NOT: PatternDecl @@ -25,6 +29,14 @@ // CHECK-CUSTOM-NEXT: PatternDecl // CHECK-CUSTOM-NOT: PatternDecl +// CHECK-NOSPLIT: Module +// CHECK-NOSPLIT-NEXT: PatternDecl +// CHECK-NOSPLIT-NOT: Module +// CHECK-NOSPLIT: PatternDecl +// CHECK-NOSPLIT-NOT: Module +// CHECK-NOSPLIT: PatternDecl +// CHECK-NOSPLIT-NOT: Module + Pattern => erase op; // ----- -- GitLab From 04a6e0f1634f9a53120c27a30250d26dff4ada1c Mon Sep 17 00:00:00 2001 From: Paul Robinson Date: Fri, 22 Mar 2024 05:35:09 -0700 Subject: [PATCH 050/306] [X86][Headers] change 'yields' to 'returns' in more places --- clang/lib/Headers/mmintrin.h | 12 +++++----- clang/lib/Headers/smmintrin.h | 4 ++-- clang/lib/Headers/xmmintrin.h | 44 +++++++++++++++++------------------ 3 files changed, 30 insertions(+), 30 deletions(-) diff --git a/clang/lib/Headers/mmintrin.h b/clang/lib/Headers/mmintrin.h index 962d24738e7a..4e154e2d8593 100644 --- a/clang/lib/Headers/mmintrin.h +++ b/clang/lib/Headers/mmintrin.h @@ -1141,7 +1141,7 @@ _mm_xor_si64(__m64 __m1, __m64 __m2) /// [8 x i8] to determine if the element of the first vector is equal to the /// corresponding element of the second vector. /// -/// The comparison yields 0 for false, 0xFF for true. +/// Each comparison returns 0 for false, 0xFF for true. /// /// \headerfile /// @@ -1163,7 +1163,7 @@ _mm_cmpeq_pi8(__m64 __m1, __m64 __m2) /// [4 x i16] to determine if the element of the first vector is equal to the /// corresponding element of the second vector. /// -/// The comparison yields 0 for false, 0xFFFF for true. +/// Each comparison returns 0 for false, 0xFFFF for true. /// /// \headerfile /// @@ -1185,7 +1185,7 @@ _mm_cmpeq_pi16(__m64 __m1, __m64 __m2) /// [2 x i32] to determine if the element of the first vector is equal to the /// corresponding element of the second vector. /// -/// The comparison yields 0 for false, 0xFFFFFFFF for true. +/// Each comparison returns 0 for false, 0xFFFFFFFF for true. /// /// \headerfile /// @@ -1207,7 +1207,7 @@ _mm_cmpeq_pi32(__m64 __m1, __m64 __m2) /// [8 x i8] to determine if the element of the first vector is greater than /// the corresponding element of the second vector. /// -/// The comparison yields 0 for false, 0xFF for true. +/// Each comparison returns 0 for false, 0xFF for true. /// /// \headerfile /// @@ -1229,7 +1229,7 @@ _mm_cmpgt_pi8(__m64 __m1, __m64 __m2) /// [4 x i16] to determine if the element of the first vector is greater than /// the corresponding element of the second vector. /// -/// The comparison yields 0 for false, 0xFFFF for true. +/// Each comparison returns 0 for false, 0xFFFF for true. /// /// \headerfile /// @@ -1251,7 +1251,7 @@ _mm_cmpgt_pi16(__m64 __m1, __m64 __m2) /// [2 x i32] to determine if the element of the first vector is greater than /// the corresponding element of the second vector. /// -/// The comparison yields 0 for false, 0xFFFFFFFF for true. +/// Each comparison returns 0 for false, 0xFFFFFFFF for true. /// /// \headerfile /// diff --git a/clang/lib/Headers/smmintrin.h b/clang/lib/Headers/smmintrin.h index 9fb9cc9b0134..b3fec474e35a 100644 --- a/clang/lib/Headers/smmintrin.h +++ b/clang/lib/Headers/smmintrin.h @@ -1188,7 +1188,7 @@ static __inline__ int __DEFAULT_FN_ATTRS _mm_testnzc_si128(__m128i __M, /// Compares each of the corresponding 64-bit values of the 128-bit /// integer vectors for equality. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. /// /// \headerfile /// @@ -2303,7 +2303,7 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_minpos_epu16(__m128i __V) { /// integer vectors to determine if the values in the first operand are /// greater than those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. /// /// \headerfile /// diff --git a/clang/lib/Headers/xmmintrin.h b/clang/lib/Headers/xmmintrin.h index 040194786a27..1ef89de9c9f5 100644 --- a/clang/lib/Headers/xmmintrin.h +++ b/clang/lib/Headers/xmmintrin.h @@ -484,7 +484,7 @@ _mm_xor_ps(__m128 __a, __m128 __b) /// Compares two 32-bit float values in the low-order bits of both /// operands for equality. /// -/// The comparison yields 0x0 for false, 0xFFFFFFFF for true, in the +/// The comparison returns 0x0 for false, 0xFFFFFFFF for true, in the /// low-order bits of a vector [4 x float]. /// If either value in a comparison is NaN, returns false. /// @@ -509,7 +509,7 @@ _mm_cmpeq_ss(__m128 __a, __m128 __b) /// Compares each of the corresponding 32-bit float values of the /// 128-bit vectors of [4 x float] for equality. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true. /// If either value in a comparison is NaN, returns false. /// /// \headerfile @@ -531,7 +531,7 @@ _mm_cmpeq_ps(__m128 __a, __m128 __b) /// operands to determine if the value in the first operand is less than the /// corresponding value in the second operand. /// -/// The comparison yields 0x0 for false, 0xFFFFFFFF for true, in the +/// The comparison returns 0x0 for false, 0xFFFFFFFF for true, in the /// low-order bits of a vector of [4 x float]. /// If either value in a comparison is NaN, returns false. /// @@ -557,7 +557,7 @@ _mm_cmplt_ss(__m128 __a, __m128 __b) /// 128-bit vectors of [4 x float] to determine if the values in the first /// operand are less than those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. /// If either value in a comparison is NaN, returns false. /// /// \headerfile @@ -579,7 +579,7 @@ _mm_cmplt_ps(__m128 __a, __m128 __b) /// operands to determine if the value in the first operand is less than or /// equal to the corresponding value in the second operand. /// -/// The comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true, in +/// The comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true, in /// the low-order bits of a vector of [4 x float]. /// If either value in a comparison is NaN, returns false. /// @@ -605,7 +605,7 @@ _mm_cmple_ss(__m128 __a, __m128 __b) /// 128-bit vectors of [4 x float] to determine if the values in the first /// operand are less than or equal to those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true. /// If either value in a comparison is NaN, returns false. /// /// \headerfile @@ -627,7 +627,7 @@ _mm_cmple_ps(__m128 __a, __m128 __b) /// operands to determine if the value in the first operand is greater than /// the corresponding value in the second operand. /// -/// The comparison yields 0x0 for false, 0xFFFFFFFF for true, in the +/// The comparison returns 0x0 for false, 0xFFFFFFFF for true, in the /// low-order bits of a vector of [4 x float]. /// If either value in a comparison is NaN, returns false. /// @@ -655,7 +655,7 @@ _mm_cmpgt_ss(__m128 __a, __m128 __b) /// 128-bit vectors of [4 x float] to determine if the values in the first /// operand are greater than those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true. /// If either value in a comparison is NaN, returns false. /// /// \headerfile @@ -677,7 +677,7 @@ _mm_cmpgt_ps(__m128 __a, __m128 __b) /// operands to determine if the value in the first operand is greater than /// or equal to the corresponding value in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true, in the +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true, in the /// low-order bits of a vector of [4 x float]. /// If either value in a comparison is NaN, returns false. /// @@ -705,7 +705,7 @@ _mm_cmpge_ss(__m128 __a, __m128 __b) /// 128-bit vectors of [4 x float] to determine if the values in the first /// operand are greater than or equal to those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. /// If either value in a comparison is NaN, returns false. /// /// \headerfile @@ -726,7 +726,7 @@ _mm_cmpge_ps(__m128 __a, __m128 __b) /// Compares two 32-bit float values in the low-order bits of both operands /// for inequality. /// -/// The comparison yields 0x0 for false, 0xFFFFFFFF for true, in the +/// The comparison returns 0x0 for false, 0xFFFFFFFF for true, in the /// low-order bits of a vector of [4 x float]. /// If either value in a comparison is NaN, returns true. /// @@ -752,7 +752,7 @@ _mm_cmpneq_ss(__m128 __a, __m128 __b) /// Compares each of the corresponding 32-bit float values of the /// 128-bit vectors of [4 x float] for inequality. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true. /// If either value in a comparison is NaN, returns true. /// /// \headerfile @@ -775,7 +775,7 @@ _mm_cmpneq_ps(__m128 __a, __m128 __b) /// operands to determine if the value in the first operand is not less than /// the corresponding value in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true, in the +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true, in the /// low-order bits of a vector of [4 x float]. /// If either value in a comparison is NaN, returns true. /// @@ -802,7 +802,7 @@ _mm_cmpnlt_ss(__m128 __a, __m128 __b) /// 128-bit vectors of [4 x float] to determine if the values in the first /// operand are not less than those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true. /// If either value in a comparison is NaN, returns true. /// /// \headerfile @@ -825,7 +825,7 @@ _mm_cmpnlt_ps(__m128 __a, __m128 __b) /// operands to determine if the value in the first operand is not less than /// or equal to the corresponding value in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true, in the +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true, in the /// low-order bits of a vector of [4 x float]. /// If either value in a comparison is NaN, returns true. /// @@ -852,7 +852,7 @@ _mm_cmpnle_ss(__m128 __a, __m128 __b) /// 128-bit vectors of [4 x float] to determine if the values in the first /// operand are not less than or equal to those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true. /// If either value in a comparison is NaN, returns true. /// /// \headerfile @@ -875,7 +875,7 @@ _mm_cmpnle_ps(__m128 __a, __m128 __b) /// operands to determine if the value in the first operand is not greater /// than the corresponding value in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true, in the +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true, in the /// low-order bits of a vector of [4 x float]. /// If either value in a comparison is NaN, returns true. /// @@ -904,7 +904,7 @@ _mm_cmpngt_ss(__m128 __a, __m128 __b) /// 128-bit vectors of [4 x float] to determine if the values in the first /// operand are not greater than those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true. /// If either value in a comparison is NaN, returns true. /// /// \headerfile @@ -927,7 +927,7 @@ _mm_cmpngt_ps(__m128 __a, __m128 __b) /// operands to determine if the value in the first operand is not greater /// than or equal to the corresponding value in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true, in the +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true, in the /// low-order bits of a vector of [4 x float]. /// If either value in a comparison is NaN, returns true. /// @@ -956,7 +956,7 @@ _mm_cmpnge_ss(__m128 __a, __m128 __b) /// 128-bit vectors of [4 x float] to determine if the values in the first /// operand are not greater than or equal to those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true. /// If either value in a comparison is NaN, returns true. /// /// \headerfile @@ -3061,7 +3061,7 @@ _mm_movemask_ps(__m128 __a) /// [4 x float], using the operation specified by the immediate integer /// operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true. /// If either value in a comparison is NaN, comparisons that are ordered /// return false, and comparisons that are unordered return true. /// @@ -3096,7 +3096,7 @@ _mm_movemask_ps(__m128 __a) /// vectors of [4 x float], using the operation specified by the immediate /// integer operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true. /// If either value in a comparison is NaN, comparisons that are ordered /// return false, and comparisons that are unordered return true. /// -- GitLab From 31a9a4b83720af79110941824abe28fc6ff42355 Mon Sep 17 00:00:00 2001 From: yronglin Date: Fri, 22 Mar 2024 20:45:17 +0800 Subject: [PATCH 051/306] [libc++] Implement LWG3528 (`make_from_tuple` can perform (the equivalent of) a C-style cast) (#85263) Implement [LWG3528](https://wg21.link/LWG3528). Based on LWG3528(https://wg21.link/LWG3528) and http://eel.is/c++draft/description#structure.requirements-9, the standard allows to impose requirements, we constraint `std::make_from_tuple` to make `std::make_from_tuple` SFINAE friendly and also avoid worse diagnostic messages. We still keep the constraints of `std::__make_from_tuple_impl` so that `std::__make_from_tuple_impl` will have the same advantages when used alone. --------- Signed-off-by: yronglin --- libcxx/docs/Status/Cxx23Issues.csv | 2 +- libcxx/include/tuple | 28 +++++- .../tuple.apply/make_from_tuple.pass.cpp | 88 +++++++++++++++++++ 3 files changed, 116 insertions(+), 2 deletions(-) diff --git a/libcxx/docs/Status/Cxx23Issues.csv b/libcxx/docs/Status/Cxx23Issues.csv index 8de265f4d1b6..ebdc4a745c9f 100644 --- a/libcxx/docs/Status/Cxx23Issues.csv +++ b/libcxx/docs/Status/Cxx23Issues.csv @@ -77,7 +77,7 @@ `3523 `__,"``iota_view::sentinel`` is not always ``iota_view``'s sentinel","June 2021","","","|ranges|" `3526 `__,"Return types of ``uses_allocator_construction_args`` unspecified","June 2021","","" `3527 `__,"``uses_allocator_construction_args`` handles rvalue pairs of rvalue references incorrectly","June 2021","","" -`3528 `__,"``make_from_tuple`` can perform (the equivalent of) a C-style cast","June 2021","","" +`3528 `__,"``make_from_tuple`` can perform (the equivalent of) a C-style cast","June 2021","|Complete|","19.0" `3529 `__,"``priority_queue(first, last)`` should construct ``c`` with ``(first, last)``","June 2021","|Complete|","14.0" `3530 `__,"``BUILTIN-PTR-MEOW`` should not opt the type out of syntactic checks","June 2021","","" `3532 `__,"``split_view::inner-iterator::operator++(int)`` should depend on ``Base``","June 2021","","","|ranges|" diff --git a/libcxx/include/tuple b/libcxx/include/tuple index f78db061b844..a9f0d680fe0e 100644 --- a/libcxx/include/tuple +++ b/libcxx/include/tuple @@ -1377,15 +1377,41 @@ inline _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) apply(_Fn&& __f, _Tuple&& std::forward<_Tuple>(__t), typename __make_tuple_indices>>::type{})) +#if _LIBCPP_STD_VER >= 20 template inline _LIBCPP_HIDE_FROM_ABI constexpr _Tp __make_from_tuple_impl(_Tuple&& __t, __tuple_indices<_Idx...>) + noexcept(noexcept(_Tp(std::get<_Idx>(std::forward<_Tuple>(__t))...))) + requires is_constructible_v<_Tp, decltype(std::get<_Idx>(std::forward<_Tuple>(__t)))...> { + return _Tp(std::get<_Idx>(std::forward<_Tuple>(__t))...); +} +#else +template +inline _LIBCPP_HIDE_FROM_ABI constexpr _Tp __make_from_tuple_impl(_Tuple&& __t, __tuple_indices<_Idx...>, + enable_if_t(std::forward<_Tuple>(__t)))...>> * = nullptr) _LIBCPP_NOEXCEPT_RETURN(_Tp(std::get<_Idx>(std::forward<_Tuple>(__t))...)) +#endif // _LIBCPP_STD_VER >= 20 + +template >>::type, class = void> +inline constexpr bool __can_make_from_tuple = false; +template +inline constexpr bool __can_make_from_tuple<_Tp, _Tuple, __tuple_indices<_Idx...>, + enable_if_t(std::declval<_Tuple>()))...>>> = true; + +// Based on LWG3528(https://wg21.link/LWG3528) and http://eel.is/c++draft/description#structure.requirements-9, +// the standard allows to impose requirements, we constraint std::make_from_tuple to make std::make_from_tuple +// SFINAE friendly and also avoid worse diagnostic messages. We still keep the constraints of std::__make_from_tuple_impl +// so that std::__make_from_tuple_impl will have the same advantages when used alone. +#if _LIBCPP_STD_VER >= 20 template + requires __can_make_from_tuple<_Tp, _Tuple> // strengthen +#else +template >> // strengthen +#endif // _LIBCPP_STD_VER >= 20 inline _LIBCPP_HIDE_FROM_ABI constexpr _Tp make_from_tuple(_Tuple&& __t) _LIBCPP_NOEXCEPT_RETURN(std::__make_from_tuple_impl<_Tp>( std::forward<_Tuple>(__t), typename __make_tuple_indices>>::type{})) - # undef _LIBCPP_NOEXCEPT_RETURN # endif // _LIBCPP_STD_VER >= 17 diff --git a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.apply/make_from_tuple.pass.cpp b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.apply/make_from_tuple.pass.cpp index e3a21149c21e..d7374351afa8 100644 --- a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.apply/make_from_tuple.pass.cpp +++ b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.apply/make_from_tuple.pass.cpp @@ -195,6 +195,94 @@ void test_noexcept() { } } +namespace LWG3528 { +template +auto test_make_from_tuple(T&&, Tuple&& t) -> decltype(std::make_from_tuple(t), uint8_t()) { + return 0; +} +template +uint32_t test_make_from_tuple(...) { + return 0; +} + +template +static constexpr bool can_make_from_tuple = + std::is_same_v(T{}, Tuple{})), uint8_t>; + +template +auto test_make_from_tuple_impl(T&&, Tuple&& t) + -> decltype(std::__make_from_tuple_impl( + t, typename std::__make_tuple_indices< std::tuple_size_v>>::type{}), + uint8_t()) { + return 0; +} +template +uint32_t test_make_from_tuple_impl(...) { + return 0; +} + +template +static constexpr bool can_make_from_tuple_impl = + std::is_same_v(T{}, Tuple{})), uint8_t>; + +struct A { + int a; +}; +struct B : public A {}; + +struct C { + C(const B&) {} +}; + +enum class D { + ONE, + TWO, +}; + +// Test std::make_from_tuple constraints. + +// reinterpret_cast +static_assert(!can_make_from_tuple>); +static_assert(can_make_from_tuple>); + +// const_cast +static_assert(!can_make_from_tuple>); +static_assert(!can_make_from_tuple>); +static_assert(can_make_from_tuple>); +static_assert(can_make_from_tuple>); +static_assert(can_make_from_tuple>); +static_assert(can_make_from_tuple>); + +// static_cast +static_assert(!can_make_from_tuple>); +static_assert(!can_make_from_tuple>); +static_assert(can_make_from_tuple>); +static_assert(can_make_from_tuple>); +static_assert(can_make_from_tuple>); + +// Test std::__make_from_tuple_impl constraints. + +// reinterpret_cast +static_assert(!can_make_from_tuple_impl>); +static_assert(can_make_from_tuple_impl>); + +// const_cast +static_assert(!can_make_from_tuple_impl>); +static_assert(!can_make_from_tuple_impl>); +static_assert(can_make_from_tuple_impl>); +static_assert(can_make_from_tuple_impl>); +static_assert(can_make_from_tuple_impl>); +static_assert(can_make_from_tuple_impl>); + +// static_cast +static_assert(!can_make_from_tuple_impl>); +static_assert(!can_make_from_tuple_impl>); +static_assert(can_make_from_tuple_impl>); +static_assert(can_make_from_tuple_impl>); +static_assert(can_make_from_tuple_impl>); + +} // namespace LWG3528 + int main(int, char**) { test_constexpr_construction(); -- GitLab From 3b3de48fd84b8269d5f45ee0a9dc6b7448368424 Mon Sep 17 00:00:00 2001 From: Amir Ayupov Date: Fri, 22 Mar 2024 06:07:17 -0700 Subject: [PATCH 052/306] [BOLT] Add BB index to BAT (#86044) --- bolt/docs/BAT.md | 11 ++--- .../bolt/Profile/BoltAddressTranslation.h | 7 +++- bolt/lib/Profile/BoltAddressTranslation.cpp | 39 ++++++++++++----- .../X86/bolt-address-translation-yaml.test | 2 +- bolt/test/X86/bolt-address-translation.test | 2 +- clang/lib/Driver/ToolChains/Clang.cpp | 4 +- clang/test/Driver/unsupported-option-gpu.c | 1 - lld/MachO/Driver.cpp | 42 +++++++++++++++++-- lld/MachO/InputSection.cpp | 38 ----------------- lld/MachO/InputSection.h | 3 -- lld/MachO/ObjC.cpp | 16 ++++--- lld/MachO/SyntheticSections.cpp | 4 +- 12 files changed, 92 insertions(+), 77 deletions(-) diff --git a/bolt/docs/BAT.md b/bolt/docs/BAT.md index 186b0e5ea89d..436593478a39 100644 --- a/bolt/docs/BAT.md +++ b/bolt/docs/BAT.md @@ -90,11 +90,12 @@ current function. ### Address translation table Delta encoding means that only the difference with the previous corresponding entry is encoded. Input offsets implicitly start at zero. -| Entry | Encoding | Description | -| ------ | ------| ----------- | -| `OutputOffset` | Continuous, Delta, ULEB128 | Function offset in output binary | -| `InputOffset` | Optional, Delta, SLEB128 | Function offset in input binary with `BRANCHENTRY` LSB bit | -| `BBHash` | Optional, 8b | Basic block entries only: basic block hash in input binary | +| Entry | Encoding | Description | Branch/BB | +| ------ | ------| ----------- | ------ | +| `OutputOffset` | Continuous, Delta, ULEB128 | Function offset in output binary | Both | +| `InputOffset` | Optional, Delta, SLEB128 | Function offset in input binary with `BRANCHENTRY` LSB bit | Both | +| `BBHash` | Optional, 8b | Basic block hash in input binary | BB | +| `BBIdx` | Optional, Delta, ULEB128 | Basic block index in input binary | BB | `BRANCHENTRY` bit denotes whether a given offset pair is a control flow source (branch or call instruction). If not set, it signifies a control flow target diff --git a/bolt/include/bolt/Profile/BoltAddressTranslation.h b/bolt/include/bolt/Profile/BoltAddressTranslation.h index 1f53f6d344ad..eda2b318f0d0 100644 --- a/bolt/include/bolt/Profile/BoltAddressTranslation.h +++ b/bolt/include/bolt/Profile/BoltAddressTranslation.h @@ -122,6 +122,10 @@ public: /// Returns BF hash by function output address (after BOLT). size_t getBFHash(uint64_t OutputAddress) const; + /// Returns BB index by function output address (after BOLT) and basic block + /// input offset. + unsigned getBBIndex(uint64_t FuncOutputAddress, uint32_t BBInputOffset) const; + /// True if a given \p Address is a function with translation table entry. bool isBATFunction(uint64_t Address) const { return Maps.count(Address); } @@ -154,7 +158,8 @@ private: std::map Maps; - using BBHashMap = std::unordered_map; + /// Map basic block input offset to a basic block index and hash pair. + using BBHashMap = std::unordered_map>; std::unordered_map> FuncHashes; /// Links outlined cold bocks to their original function diff --git a/bolt/lib/Profile/BoltAddressTranslation.cpp b/bolt/lib/Profile/BoltAddressTranslation.cpp index 1d61a1b735b4..8fe976cc00e5 100644 --- a/bolt/lib/Profile/BoltAddressTranslation.cpp +++ b/bolt/lib/Profile/BoltAddressTranslation.cpp @@ -45,6 +45,8 @@ void BoltAddressTranslation::writeEntriesForBB(MapTy &Map, LLVM_DEBUG(dbgs() << formatv(" Hash: {0:x}\n", getBBHash(HotFuncAddress, BBInputOffset))); (void)HotFuncAddress; + LLVM_DEBUG(dbgs() << formatv(" Index: {0}\n", + getBBIndex(HotFuncAddress, BBInputOffset))); // In case of conflicts (same Key mapping to different Vals), the last // update takes precedence. Of course it is not ideal to have conflicts and // those happen when we have an empty BB that either contained only @@ -217,6 +219,7 @@ void BoltAddressTranslation::writeMaps(std::map &Maps, } size_t Index = 0; uint64_t InOffset = 0; + size_t PrevBBIndex = 0; // Output and Input addresses and delta-encoded for (std::pair &KeyVal : Map) { const uint64_t OutputAddress = KeyVal.first + Address; @@ -226,11 +229,15 @@ void BoltAddressTranslation::writeMaps(std::map &Maps, encodeSLEB128(KeyVal.second - InOffset, OS); InOffset = KeyVal.second; // Keeping InOffset as if BRANCHENTRY is encoded if ((InOffset & BRANCHENTRY) == 0) { - // Basic block hash - size_t BBHash = FuncHashPair.second[InOffset >> 1]; + unsigned BBIndex; + size_t BBHash; + std::tie(BBIndex, BBHash) = FuncHashPair.second[InOffset >> 1]; OS.write(reinterpret_cast(&BBHash), 8); - LLVM_DEBUG(dbgs() << formatv("{0:x} -> {1:x} {2:x}\n", KeyVal.first, - InOffset >> 1, BBHash)); + // Basic block index in the input binary + encodeULEB128(BBIndex - PrevBBIndex, OS); + PrevBBIndex = BBIndex; + LLVM_DEBUG(dbgs() << formatv("{0:x} -> {1:x} {2:x} {3}\n", KeyVal.first, + InOffset >> 1, BBHash, BBIndex)); } } } @@ -316,6 +323,7 @@ void BoltAddressTranslation::parseMaps(std::vector &HotFuncs, LLVM_DEBUG(dbgs() << "Parsing " << NumEntries << " entries for 0x" << Twine::utohexstr(Address) << "\n"); uint64_t InputOffset = 0; + size_t BBIndex = 0; for (uint32_t J = 0; J < NumEntries; ++J) { const uint64_t OutputDelta = DE.getULEB128(&Offset, &Err); const uint64_t OutputAddress = PrevAddress + OutputDelta; @@ -330,19 +338,25 @@ void BoltAddressTranslation::parseMaps(std::vector &HotFuncs, } Map.insert(std::pair(OutputOffset, InputOffset)); size_t BBHash = 0; + size_t BBIndexDelta = 0; const bool IsBranchEntry = InputOffset & BRANCHENTRY; if (!IsBranchEntry) { BBHash = DE.getU64(&Offset, &Err); + BBIndexDelta = DE.getULEB128(&Offset, &Err); + BBIndex += BBIndexDelta; // Map basic block hash to hot fragment by input offset - FuncHashes[HotAddress].second.emplace(InputOffset >> 1, BBHash); + FuncHashes[HotAddress].second.emplace(InputOffset >> 1, + std::pair(BBIndex, BBHash)); } LLVM_DEBUG({ dbgs() << formatv( "{0:x} -> {1:x} ({2}/{3}b -> {4}/{5}b), {6:x}", OutputOffset, InputOffset, OutputDelta, getULEB128Size(OutputDelta), InputDelta, (J < EqualElems) ? 0 : getSLEB128Size(InputDelta), OutputAddress); - if (BBHash) - dbgs() << formatv(" {0:x}", BBHash); + if (!IsBranchEntry) { + dbgs() << formatv(" {0:x} {1}/{2}b", BBHash, BBIndex, + getULEB128Size(BBIndexDelta)); + } dbgs() << '\n'; }); } @@ -494,14 +508,19 @@ void BoltAddressTranslation::saveMetadata(BinaryContext &BC) { FuncHashes[BF.getAddress()].first = BF.computeHash(); BF.computeBlockHashes(); for (const BinaryBasicBlock &BB : BF) - FuncHashes[BF.getAddress()].second.emplace(BB.getInputOffset(), - BB.getHash()); + FuncHashes[BF.getAddress()].second.emplace( + BB.getInputOffset(), std::pair(BB.getIndex(), BB.getHash())); } } +unsigned BoltAddressTranslation::getBBIndex(uint64_t FuncOutputAddress, + uint32_t BBInputOffset) const { + return FuncHashes.at(FuncOutputAddress).second.at(BBInputOffset).first; +} + size_t BoltAddressTranslation::getBBHash(uint64_t FuncOutputAddress, uint32_t BBInputOffset) const { - return FuncHashes.at(FuncOutputAddress).second.at(BBInputOffset); + return FuncHashes.at(FuncOutputAddress).second.at(BBInputOffset).second; } size_t BoltAddressTranslation::getBFHash(uint64_t OutputAddress) const { diff --git a/bolt/test/X86/bolt-address-translation-yaml.test b/bolt/test/X86/bolt-address-translation-yaml.test index 25ff4e7fbfcc..4516a662697a 100644 --- a/bolt/test/X86/bolt-address-translation-yaml.test +++ b/bolt/test/X86/bolt-address-translation-yaml.test @@ -18,7 +18,7 @@ RUN: | FileCheck --check-prefix CHECK-BOLT-YAML %s WRITE-BAT-CHECK: BOLT-INFO: Wrote 5 BAT maps WRITE-BAT-CHECK: BOLT-INFO: Wrote 4 function and 22 basic block hashes -WRITE-BAT-CHECK: BOLT-INFO: BAT section size (bytes): 344 +WRITE-BAT-CHECK: BOLT-INFO: BAT section size (bytes): 376 READ-BAT-CHECK-NOT: BOLT-ERROR: unable to save profile in YAML format for input file processed by BOLT READ-BAT-CHECK: BOLT-INFO: Parsed 5 BAT entries diff --git a/bolt/test/X86/bolt-address-translation.test b/bolt/test/X86/bolt-address-translation.test index 4277b4e0d0fe..5c1db89e3c6b 100644 --- a/bolt/test/X86/bolt-address-translation.test +++ b/bolt/test/X86/bolt-address-translation.test @@ -37,7 +37,7 @@ # CHECK: BOLT: 3 out of 7 functions were overwritten. # CHECK: BOLT-INFO: Wrote 6 BAT maps # CHECK: BOLT-INFO: Wrote 3 function and 58 basic block hashes -# CHECK: BOLT-INFO: BAT section size (bytes): 816 +# CHECK: BOLT-INFO: BAT section size (bytes): 920 # # usqrt mappings (hot part). We match against any key (left side containing # the bolted binary offsets) because BOLT may change where it puts instructions diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp index 86a287db72a4..bc9cc8ce6cf5 100644 --- a/clang/lib/Driver/ToolChains/Clang.cpp +++ b/clang/lib/Driver/ToolChains/Clang.cpp @@ -5863,8 +5863,8 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA, } else if (Triple.getArch() == llvm::Triple::x86_64) { Ok = llvm::is_contained({"small", "kernel", "medium", "large", "tiny"}, CM); - } else if (Triple.isNVPTX() || Triple.isAMDGPU() || Triple.isSPIRV()) { - // NVPTX/AMDGPU/SPIRV does not care about the code model and will accept + } else if (Triple.isNVPTX() || Triple.isAMDGPU()) { + // NVPTX/AMDGPU does not care about the code model and will accept // whatever works for the host. Ok = true; } else if (Triple.isSPARC64()) { diff --git a/clang/test/Driver/unsupported-option-gpu.c b/clang/test/Driver/unsupported-option-gpu.c index 5618b2cba72e..f23cb71ebfb0 100644 --- a/clang/test/Driver/unsupported-option-gpu.c +++ b/clang/test/Driver/unsupported-option-gpu.c @@ -2,5 +2,4 @@ // DEFINE: %{check} = %clang -### --target=x86_64-linux-gnu -c -mcmodel=medium // RUN: %{check} -x cuda %s --cuda-path=%S/Inputs/CUDA/usr/local/cuda --offload-arch=sm_60 --no-cuda-version-check -fbasic-block-sections=all -// RUN: %{check} -x hip %s --offload=spirv64 -nogpulib -nogpuinc // RUN: %{check} -x hip %s --rocm-path=%S/Inputs/rocm -nogpulib -nogpuinc diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp index 919a14b8bcf0..36248925d65a 100644 --- a/lld/MachO/Driver.cpp +++ b/lld/MachO/Driver.cpp @@ -612,7 +612,7 @@ static void replaceCommonSymbols() { if (!osec) osec = ConcatOutputSection::getOrCreateForInput(isec); isec->parent = osec; - addInputSection(isec); + inputSections.push_back(isec); // FIXME: CommonSymbol should store isReferencedDynamically, noDeadStrip // and pass them on here. @@ -1220,18 +1220,53 @@ static void createFiles(const InputArgList &args) { static void gatherInputSections() { TimeTraceScope timeScope("Gathering input sections"); + int inputOrder = 0; for (const InputFile *file : inputFiles) { for (const Section *section : file->sections) { // Compact unwind entries require special handling elsewhere. (In // contrast, EH frames are handled like regular ConcatInputSections.) if (section->name == section_names::compactUnwind) continue; - for (const Subsection &subsection : section->subsections) - addInputSection(subsection.isec); + ConcatOutputSection *osec = nullptr; + for (const Subsection &subsection : section->subsections) { + if (auto *isec = dyn_cast(subsection.isec)) { + if (isec->isCoalescedWeak()) + continue; + if (config->emitInitOffsets && + sectionType(isec->getFlags()) == S_MOD_INIT_FUNC_POINTERS) { + in.initOffsets->addInput(isec); + continue; + } + isec->outSecOff = inputOrder++; + if (!osec) + osec = ConcatOutputSection::getOrCreateForInput(isec); + isec->parent = osec; + inputSections.push_back(isec); + } else if (auto *isec = + dyn_cast(subsection.isec)) { + if (isec->getName() == section_names::objcMethname) { + if (in.objcMethnameSection->inputOrder == UnspecifiedInputOrder) + in.objcMethnameSection->inputOrder = inputOrder++; + in.objcMethnameSection->addInput(isec); + } else { + if (in.cStringSection->inputOrder == UnspecifiedInputOrder) + in.cStringSection->inputOrder = inputOrder++; + in.cStringSection->addInput(isec); + } + } else if (auto *isec = + dyn_cast(subsection.isec)) { + if (in.wordLiteralSection->inputOrder == UnspecifiedInputOrder) + in.wordLiteralSection->inputOrder = inputOrder++; + in.wordLiteralSection->addInput(isec); + } else { + llvm_unreachable("unexpected input section kind"); + } + } } if (!file->objCImageInfo.empty()) in.objCImageInfo->addFile(file); } + assert(inputOrder <= UnspecifiedInputOrder); } static void foldIdenticalLiterals() { @@ -1387,7 +1422,6 @@ bool link(ArrayRef argsArr, llvm::raw_ostream &stdoutOS, concatOutputSections.clear(); inputFiles.clear(); inputSections.clear(); - inputSectionsOrder = 0; loadedArchives.clear(); loadedObjectFrameworks.clear(); missingAutolinkWarnings.clear(); diff --git a/lld/MachO/InputSection.cpp b/lld/MachO/InputSection.cpp index 22930d52dd1d..8f5affb1dc21 100644 --- a/lld/MachO/InputSection.cpp +++ b/lld/MachO/InputSection.cpp @@ -37,44 +37,6 @@ static_assert(sizeof(void *) != 8 || "instances of it"); std::vector macho::inputSections; -int macho::inputSectionsOrder = 0; - -// Call this function to add a new InputSection and have it routed to the -// appropriate container. Depending on its type and current config, it will -// either be added to 'inputSections' vector or to a synthetic section. -void lld::macho::addInputSection(InputSection *inputSection) { - if (auto *isec = dyn_cast(inputSection)) { - if (isec->isCoalescedWeak()) - return; - if (config->emitInitOffsets && - sectionType(isec->getFlags()) == S_MOD_INIT_FUNC_POINTERS) { - in.initOffsets->addInput(isec); - return; - } - isec->outSecOff = inputSectionsOrder++; - auto *osec = ConcatOutputSection::getOrCreateForInput(isec); - isec->parent = osec; - inputSections.push_back(isec); - } else if (auto *isec = dyn_cast(inputSection)) { - if (isec->getName() == section_names::objcMethname) { - if (in.objcMethnameSection->inputOrder == UnspecifiedInputOrder) - in.objcMethnameSection->inputOrder = inputSectionsOrder++; - in.objcMethnameSection->addInput(isec); - } else { - if (in.cStringSection->inputOrder == UnspecifiedInputOrder) - in.cStringSection->inputOrder = inputSectionsOrder++; - in.cStringSection->addInput(isec); - } - } else if (auto *isec = dyn_cast(inputSection)) { - if (in.wordLiteralSection->inputOrder == UnspecifiedInputOrder) - in.wordLiteralSection->inputOrder = inputSectionsOrder++; - in.wordLiteralSection->addInput(isec); - } else { - llvm_unreachable("unexpected input section kind"); - } - - assert(inputSectionsOrder <= UnspecifiedInputOrder); -} uint64_t InputSection::getFileSize() const { return isZeroFill(getFlags()) ? 0 : getSize(); diff --git a/lld/MachO/InputSection.h b/lld/MachO/InputSection.h index 694bdf734907..b25f0638f4c6 100644 --- a/lld/MachO/InputSection.h +++ b/lld/MachO/InputSection.h @@ -302,8 +302,6 @@ bool isEhFrameSection(const InputSection *); bool isGccExceptTabSection(const InputSection *); extern std::vector inputSections; -// This is used as a counter for specyfing input order for input sections -extern int inputSectionsOrder; namespace section_names { @@ -371,7 +369,6 @@ constexpr const char addrSig[] = "__llvm_addrsig"; } // namespace section_names -void addInputSection(InputSection *inputSection); } // namespace macho std::string toString(const macho::InputSection *); diff --git a/lld/MachO/ObjC.cpp b/lld/MachO/ObjC.cpp index 5902b82d30f5..40df2243b26f 100644 --- a/lld/MachO/ObjC.cpp +++ b/lld/MachO/ObjC.cpp @@ -790,7 +790,7 @@ void ObjcCategoryMerger::emitAndLinkProtocolList( infoCategoryWriter.catPtrListInfo.align); listSec->parent = infoCategoryWriter.catPtrListInfo.outputSection; listSec->live = true; - addInputSection(listSec); + allInputSections.push_back(listSec); listSec->parent = infoCategoryWriter.catPtrListInfo.outputSection; @@ -848,7 +848,7 @@ void ObjcCategoryMerger::emitAndLinkPointerList( infoCategoryWriter.catPtrListInfo.align); listSec->parent = infoCategoryWriter.catPtrListInfo.outputSection; listSec->live = true; - addInputSection(listSec); + allInputSections.push_back(listSec); listSec->parent = infoCategoryWriter.catPtrListInfo.outputSection; @@ -889,7 +889,7 @@ ObjcCategoryMerger::emitCatListEntrySec(const std::string &forCateogryName, bodyData, infoCategoryWriter.catListInfo.align); newCatList->parent = infoCategoryWriter.catListInfo.outputSection; newCatList->live = true; - addInputSection(newCatList); + allInputSections.push_back(newCatList); newCatList->parent = infoCategoryWriter.catListInfo.outputSection; @@ -927,7 +927,7 @@ Defined *ObjcCategoryMerger::emitCategoryBody(const std::string &name, bodyData, infoCategoryWriter.catBodyInfo.align); newBodySec->parent = infoCategoryWriter.catBodyInfo.outputSection; newBodySec->live = true; - addInputSection(newBodySec); + allInputSections.push_back(newBodySec); std::string symName = objc::symbol_names::category + baseClassName + "_$_(" + name + ")"; @@ -1132,7 +1132,7 @@ void ObjcCategoryMerger::generateCatListForNonErasedCategories( infoCategoryWriter.catListInfo.align); listSec->parent = infoCategoryWriter.catListInfo.outputSection; listSec->live = true; - addInputSection(listSec); + allInputSections.push_back(listSec); std::string slotSymName = "<__objc_catlist slot for category "; slotSymName += nonErasedCatBody->getName(); @@ -1221,11 +1221,9 @@ void ObjcCategoryMerger::doCleanup() { generatedSectionData.clear(); } StringRef ObjcCategoryMerger::newStringData(const char *str) { uint32_t len = strlen(str); - uint32_t bufSize = len + 1; - auto &data = newSectionData(bufSize); + auto &data = newSectionData(len + 1); char *strData = reinterpret_cast(data.data()); - // Copy the string chars and null-terminator - memcpy(strData, str, bufSize); + strncpy(strData, str, len); return StringRef(strData, len); } diff --git a/lld/MachO/SyntheticSections.cpp b/lld/MachO/SyntheticSections.cpp index 1b3694528de1..7ee3261ce307 100644 --- a/lld/MachO/SyntheticSections.cpp +++ b/lld/MachO/SyntheticSections.cpp @@ -793,7 +793,7 @@ void StubHelperSection::setUp() { in.imageLoaderCache->parent = ConcatOutputSection::getOrCreateForInput(in.imageLoaderCache); - addInputSection(in.imageLoaderCache); + inputSections.push_back(in.imageLoaderCache); // Since this isn't in the symbol table or in any input file, the noDeadStrip // argument doesn't matter. dyldPrivate = @@ -855,7 +855,7 @@ ConcatInputSection *ObjCSelRefsSection::makeSelRef(StringRef methname) { /*addend=*/static_cast(methnameOffset), /*referent=*/in.objcMethnameSection->isec}); objcSelref->parent = ConcatOutputSection::getOrCreateForInput(objcSelref); - addInputSection(objcSelref); + inputSections.push_back(objcSelref); objcSelref->isFinal = true; methnameToSelref[CachedHashStringRef(methname)] = objcSelref; return objcSelref; -- GitLab From b3f98dff75469b115e3d4b1f10cbf270c8ee81af Mon Sep 17 00:00:00 2001 From: Orlando Cazalet-Hyams Date: Fri, 22 Mar 2024 13:52:11 +0000 Subject: [PATCH 053/306] [RemoveDIs] Load into new debug info format by default in llvm-lto and llvm-lto2 (#86271) Directly load all bitcode into the new debug info format in `llvm-lto` and `llvm-lto2`. This means that new-mode bitcode no longer round-trips back to old-mode after parsing, and that old-mode bitcode gets auto-upgraded to new-mode debug info (which is the current in-memory default in LLVM). --- llvm/lib/LTO/LTO.cpp | 4 +++- llvm/tools/llvm-lto/llvm-lto.cpp | 4 ++++ llvm/tools/llvm-lto2/llvm-lto2.cpp | 4 ++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp index b58418c64a11..53060df7f503 100644 --- a/llvm/lib/LTO/LTO.cpp +++ b/llvm/lib/LTO/LTO.cpp @@ -579,7 +579,9 @@ LTO::RegularLTOState::RegularLTOState(unsigned ParallelCodeGenParallelismLevel, const Config &Conf) : ParallelCodeGenParallelismLevel(ParallelCodeGenParallelismLevel), Ctx(Conf), CombinedModule(std::make_unique("ld-temp.o", Ctx)), - Mover(std::make_unique(*CombinedModule)) {} + Mover(std::make_unique(*CombinedModule)) { + CombinedModule->IsNewDbgInfoFormat = UseNewDbgInfoFormat; +} LTO::ThinLTOState::ThinLTOState(ThinBackend Backend) : Backend(Backend), CombinedIndex(/*HaveGVs*/ false) { diff --git a/llvm/tools/llvm-lto/llvm-lto.cpp b/llvm/tools/llvm-lto/llvm-lto.cpp index 7943d6952b82..3c452b650cee 100644 --- a/llvm/tools/llvm-lto/llvm-lto.cpp +++ b/llvm/tools/llvm-lto/llvm-lto.cpp @@ -270,6 +270,7 @@ static cl::opt TryUseNewDbgInfoFormat( cl::init(false), cl::Hidden); extern cl::opt UseNewDbgInfoFormat; +extern cl::opt LoadBitcodeIntoNewDbgInfoFormat; namespace { @@ -943,6 +944,9 @@ int main(int argc, char **argv) { InitLLVM X(argc, argv); cl::HideUnrelatedOptions({<OCategory, &getColorCategory()}); cl::ParseCommandLineOptions(argc, argv, "llvm LTO linker\n"); + // Load bitcode into the new debug info format by default. + if (LoadBitcodeIntoNewDbgInfoFormat == cl::boolOrDefault::BOU_UNSET) + LoadBitcodeIntoNewDbgInfoFormat = cl::boolOrDefault::BOU_TRUE; // RemoveDIs debug-info transition: tests may request that we /try/ to use the // new debug-info format. diff --git a/llvm/tools/llvm-lto2/llvm-lto2.cpp b/llvm/tools/llvm-lto2/llvm-lto2.cpp index d5de4f6b1a27..f222d02bd7ce 100644 --- a/llvm/tools/llvm-lto2/llvm-lto2.cpp +++ b/llvm/tools/llvm-lto2/llvm-lto2.cpp @@ -193,6 +193,7 @@ static cl::opt TryUseNewDbgInfoFormat( cl::init(false), cl::Hidden); extern cl::opt UseNewDbgInfoFormat; +extern cl::opt LoadBitcodeIntoNewDbgInfoFormat; static void check(Error E, std::string Msg) { if (!E) @@ -228,6 +229,9 @@ static int usage() { static int run(int argc, char **argv) { cl::ParseCommandLineOptions(argc, argv, "Resolution-based LTO test harness"); + // Load bitcode into the new debug info format by default. + if (LoadBitcodeIntoNewDbgInfoFormat == cl::boolOrDefault::BOU_UNSET) + LoadBitcodeIntoNewDbgInfoFormat = cl::boolOrDefault::BOU_TRUE; // RemoveDIs debug-info transition: tests may request that we /try/ to use the // new debug-info format. -- GitLab From a277dd82d89a17849cf99fb335660ea0b8894878 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Fri, 22 Mar 2024 13:43:58 +0000 Subject: [PATCH 054/306] [X86] vector-half-conversions.ll - add v4f16->v4i32 fptosi/fptoui test coverage --- .../CodeGen/X86/vector-half-conversions.ll | 254 ++++++++++++++++++ 1 file changed, 254 insertions(+) diff --git a/llvm/test/CodeGen/X86/vector-half-conversions.ll b/llvm/test/CodeGen/X86/vector-half-conversions.ll index ba21af231985..563cf0165013 100644 --- a/llvm/test/CodeGen/X86/vector-half-conversions.ll +++ b/llvm/test/CodeGen/X86/vector-half-conversions.ll @@ -4989,3 +4989,257 @@ define <4 x i32> @fptosi_2f16_to_4i32(<2 x half> %a) nounwind { %ext = shufflevector <2 x i32> %cvt, <2 x i32> zeroinitializer, <4 x i32> ret <4 x i32> %ext } + +define <4 x i32> @fptosi_4f16_to_4i32(<4 x half> %a) nounwind { +; AVX-LABEL: fptosi_4f16_to_4i32: +; AVX: # %bb.0: +; AVX-NEXT: subq $72, %rsp +; AVX-NEXT: vmovdqa %xmm0, %xmm1 +; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX-NEXT: vpsrld $16, %xmm0, %xmm0 +; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX-NEXT: vmovshdup {{.*#+}} xmm0 = xmm1[1,1,3,3] +; AVX-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill +; AVX-NEXT: vpsrlq $48, %xmm1, %xmm0 +; AVX-NEXT: callq __extendhfsf2@PLT +; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX-NEXT: vmovaps (%rsp), %xmm0 # 16-byte Reload +; AVX-NEXT: callq __extendhfsf2@PLT +; AVX-NEXT: vinsertps $16, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload +; AVX-NEXT: # xmm0 = xmm0[0],mem[0],xmm0[2,3] +; AVX-NEXT: vcvttps2dq %xmm0, %xmm0 +; AVX-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; AVX-NEXT: callq __extendhfsf2@PLT +; AVX-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; AVX-NEXT: callq __extendhfsf2@PLT +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload +; AVX-NEXT: vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3] +; AVX-NEXT: vcvttps2dq %xmm0, %xmm0 +; AVX-NEXT: vunpcklpd (%rsp), %xmm0, %xmm0 # 16-byte Folded Reload +; AVX-NEXT: # xmm0 = xmm0[0],mem[0] +; AVX-NEXT: addq $72, %rsp +; AVX-NEXT: retq +; +; F16C-LABEL: fptosi_4f16_to_4i32: +; F16C: # %bb.0: +; F16C-NEXT: vcvtph2ps %xmm0, %ymm0 +; F16C-NEXT: vcvttps2dq %ymm0, %ymm0 +; F16C-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0 +; F16C-NEXT: vzeroupper +; F16C-NEXT: retq +; +; AVX512-LABEL: fptosi_4f16_to_4i32: +; AVX512: # %bb.0: +; AVX512-NEXT: vcvtph2ps %xmm0, %ymm0 +; AVX512-NEXT: vcvttps2dq %ymm0, %ymm0 +; AVX512-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0 +; AVX512-NEXT: vzeroupper +; AVX512-NEXT: retq + %cvt = fptosi <4 x half> %a to <4 x i32> + ret <4 x i32> %cvt +} + +define <4 x i32> @fptoui_2f16_to_4i32(<2 x half> %a) nounwind { +; AVX1-LABEL: fptoui_2f16_to_4i32: +; AVX1: # %bb.0: +; AVX1-NEXT: subq $40, %rsp +; AVX1-NEXT: vpsrld $16, %xmm0, %xmm1 +; AVX1-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX1-NEXT: callq __extendhfsf2@PLT +; AVX1-NEXT: vmovdqa %xmm0, (%rsp) # 16-byte Spill +; AVX1-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; AVX1-NEXT: callq __extendhfsf2@PLT +; AVX1-NEXT: vmovaps (%rsp), %xmm1 # 16-byte Reload +; AVX1-NEXT: vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3] +; AVX1-NEXT: vcvttps2dq %xmm0, %xmm1 +; AVX1-NEXT: vpsrad $31, %xmm1, %xmm2 +; AVX1-NEXT: vsubps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; AVX1-NEXT: vcvttps2dq %xmm0, %xmm0 +; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vpor %xmm0, %xmm1, %xmm0 +; AVX1-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero +; AVX1-NEXT: addq $40, %rsp +; AVX1-NEXT: retq +; +; AVX2-LABEL: fptoui_2f16_to_4i32: +; AVX2: # %bb.0: +; AVX2-NEXT: subq $40, %rsp +; AVX2-NEXT: vpsrld $16, %xmm0, %xmm1 +; AVX2-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX2-NEXT: callq __extendhfsf2@PLT +; AVX2-NEXT: vmovdqa %xmm0, (%rsp) # 16-byte Spill +; AVX2-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; AVX2-NEXT: callq __extendhfsf2@PLT +; AVX2-NEXT: vmovaps (%rsp), %xmm1 # 16-byte Reload +; AVX2-NEXT: vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3] +; AVX2-NEXT: vcvttps2dq %xmm0, %xmm1 +; AVX2-NEXT: vpsrad $31, %xmm1, %xmm2 +; AVX2-NEXT: vbroadcastss {{.*#+}} xmm3 = [2.14748365E+9,2.14748365E+9,2.14748365E+9,2.14748365E+9] +; AVX2-NEXT: vsubps %xmm3, %xmm0, %xmm0 +; AVX2-NEXT: vcvttps2dq %xmm0, %xmm0 +; AVX2-NEXT: vpand %xmm2, %xmm0, %xmm0 +; AVX2-NEXT: vpor %xmm0, %xmm1, %xmm0 +; AVX2-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero +; AVX2-NEXT: addq $40, %rsp +; AVX2-NEXT: retq +; +; F16C-LABEL: fptoui_2f16_to_4i32: +; F16C: # %bb.0: +; F16C-NEXT: vpsrld $16, %xmm0, %xmm1 +; F16C-NEXT: vcvtph2ps %xmm1, %xmm1 +; F16C-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero +; F16C-NEXT: vcvtph2ps %xmm0, %xmm0 +; F16C-NEXT: vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; F16C-NEXT: vcvttps2dq %xmm0, %xmm1 +; F16C-NEXT: vpsrad $31, %xmm1, %xmm2 +; F16C-NEXT: vsubps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; F16C-NEXT: vcvttps2dq %xmm0, %xmm0 +; F16C-NEXT: vpand %xmm2, %xmm0, %xmm0 +; F16C-NEXT: vpor %xmm0, %xmm1, %xmm0 +; F16C-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero +; F16C-NEXT: retq +; +; AVX512F-LABEL: fptoui_2f16_to_4i32: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vpsrld $16, %xmm0, %xmm1 +; AVX512F-NEXT: vcvtph2ps %xmm1, %xmm1 +; AVX512F-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero +; AVX512F-NEXT: vcvtph2ps %xmm0, %xmm0 +; AVX512F-NEXT: vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; AVX512F-NEXT: vcvttps2udq %zmm0, %zmm0 +; AVX512F-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero +; AVX512F-NEXT: vzeroupper +; AVX512F-NEXT: retq +; +; AVX512-FASTLANE-LABEL: fptoui_2f16_to_4i32: +; AVX512-FASTLANE: # %bb.0: +; AVX512-FASTLANE-NEXT: vpsrld $16, %xmm0, %xmm1 +; AVX512-FASTLANE-NEXT: vcvtph2ps %xmm1, %xmm1 +; AVX512-FASTLANE-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero +; AVX512-FASTLANE-NEXT: vcvtph2ps %xmm0, %xmm0 +; AVX512-FASTLANE-NEXT: vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; AVX512-FASTLANE-NEXT: vcvttps2udq %xmm0, %xmm0 +; AVX512-FASTLANE-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero +; AVX512-FASTLANE-NEXT: retq + %cvt = fptoui <2 x half> %a to <2 x i32> + %ext = shufflevector <2 x i32> %cvt, <2 x i32> zeroinitializer, <4 x i32> + ret <4 x i32> %ext +} + +define <4 x i32> @fptoui_4f16_to_4i32(<4 x half> %a) nounwind { +; AVX1-LABEL: fptoui_4f16_to_4i32: +; AVX1: # %bb.0: +; AVX1-NEXT: subq $72, %rsp +; AVX1-NEXT: vmovdqa %xmm0, %xmm1 +; AVX1-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX1-NEXT: vpsrld $16, %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX1-NEXT: vmovshdup {{.*#+}} xmm0 = xmm1[1,1,3,3] +; AVX1-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill +; AVX1-NEXT: vpsrlq $48, %xmm1, %xmm0 +; AVX1-NEXT: callq __extendhfsf2@PLT +; AVX1-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX1-NEXT: vmovaps (%rsp), %xmm0 # 16-byte Reload +; AVX1-NEXT: callq __extendhfsf2@PLT +; AVX1-NEXT: vinsertps $16, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload +; AVX1-NEXT: # xmm0 = xmm0[0],mem[0],xmm0[2,3] +; AVX1-NEXT: vcvttps2dq %xmm0, %xmm1 +; AVX1-NEXT: vpsrad $31, %xmm1, %xmm2 +; AVX1-NEXT: vsubps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; AVX1-NEXT: vcvttps2dq %xmm0, %xmm0 +; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vpor %xmm0, %xmm1, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rsp) # 16-byte Spill +; AVX1-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; AVX1-NEXT: callq __extendhfsf2@PLT +; AVX1-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX1-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; AVX1-NEXT: callq __extendhfsf2@PLT +; AVX1-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload +; AVX1-NEXT: vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3] +; AVX1-NEXT: vcvttps2dq %xmm0, %xmm1 +; AVX1-NEXT: vpsrad $31, %xmm1, %xmm2 +; AVX1-NEXT: vsubps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; AVX1-NEXT: vcvttps2dq %xmm0, %xmm0 +; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vpor %xmm0, %xmm1, %xmm0 +; AVX1-NEXT: vpunpcklqdq (%rsp), %xmm0, %xmm0 # 16-byte Folded Reload +; AVX1-NEXT: # xmm0 = xmm0[0],mem[0] +; AVX1-NEXT: addq $72, %rsp +; AVX1-NEXT: retq +; +; AVX2-LABEL: fptoui_4f16_to_4i32: +; AVX2: # %bb.0: +; AVX2-NEXT: subq $72, %rsp +; AVX2-NEXT: vmovdqa %xmm0, %xmm1 +; AVX2-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX2-NEXT: vpsrld $16, %xmm0, %xmm0 +; AVX2-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX2-NEXT: vmovshdup {{.*#+}} xmm0 = xmm1[1,1,3,3] +; AVX2-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill +; AVX2-NEXT: vpsrlq $48, %xmm1, %xmm0 +; AVX2-NEXT: callq __extendhfsf2@PLT +; AVX2-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX2-NEXT: vmovaps (%rsp), %xmm0 # 16-byte Reload +; AVX2-NEXT: callq __extendhfsf2@PLT +; AVX2-NEXT: vinsertps $16, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload +; AVX2-NEXT: # xmm0 = xmm0[0],mem[0],xmm0[2,3] +; AVX2-NEXT: vcvttps2dq %xmm0, %xmm1 +; AVX2-NEXT: vpsrad $31, %xmm1, %xmm2 +; AVX2-NEXT: vbroadcastss {{.*#+}} xmm3 = [2.14748365E+9,2.14748365E+9,2.14748365E+9,2.14748365E+9] +; AVX2-NEXT: vsubps %xmm3, %xmm0, %xmm0 +; AVX2-NEXT: vcvttps2dq %xmm0, %xmm0 +; AVX2-NEXT: vpand %xmm2, %xmm0, %xmm0 +; AVX2-NEXT: vpor %xmm0, %xmm1, %xmm0 +; AVX2-NEXT: vmovdqa %xmm0, (%rsp) # 16-byte Spill +; AVX2-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; AVX2-NEXT: callq __extendhfsf2@PLT +; AVX2-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX2-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; AVX2-NEXT: callq __extendhfsf2@PLT +; AVX2-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload +; AVX2-NEXT: vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3] +; AVX2-NEXT: vcvttps2dq %xmm0, %xmm1 +; AVX2-NEXT: vpsrad $31, %xmm1, %xmm2 +; AVX2-NEXT: vbroadcastss {{.*#+}} xmm3 = [2.14748365E+9,2.14748365E+9,2.14748365E+9,2.14748365E+9] +; AVX2-NEXT: vsubps %xmm3, %xmm0, %xmm0 +; AVX2-NEXT: vcvttps2dq %xmm0, %xmm0 +; AVX2-NEXT: vpand %xmm2, %xmm0, %xmm0 +; AVX2-NEXT: vpor %xmm0, %xmm1, %xmm0 +; AVX2-NEXT: vpunpcklqdq (%rsp), %xmm0, %xmm0 # 16-byte Folded Reload +; AVX2-NEXT: # xmm0 = xmm0[0],mem[0] +; AVX2-NEXT: addq $72, %rsp +; AVX2-NEXT: retq +; +; F16C-LABEL: fptoui_4f16_to_4i32: +; F16C: # %bb.0: +; F16C-NEXT: vcvtph2ps %xmm0, %ymm0 +; F16C-NEXT: vcvttps2dq %ymm0, %ymm1 +; F16C-NEXT: vsubps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 +; F16C-NEXT: vcvttps2dq %ymm0, %ymm0 +; F16C-NEXT: vorps %ymm0, %ymm1, %ymm0 +; F16C-NEXT: vblendvps %ymm1, %ymm0, %ymm1, %ymm0 +; F16C-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0 +; F16C-NEXT: vzeroupper +; F16C-NEXT: retq +; +; AVX512F-LABEL: fptoui_4f16_to_4i32: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vcvtph2ps %xmm0, %ymm0 +; AVX512F-NEXT: vcvttps2udq %zmm0, %zmm0 +; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0 +; AVX512F-NEXT: vzeroupper +; AVX512F-NEXT: retq +; +; AVX512-FASTLANE-LABEL: fptoui_4f16_to_4i32: +; AVX512-FASTLANE: # %bb.0: +; AVX512-FASTLANE-NEXT: vcvtph2ps %xmm0, %ymm0 +; AVX512-FASTLANE-NEXT: vcvttps2udq %ymm0, %ymm0 +; AVX512-FASTLANE-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0 +; AVX512-FASTLANE-NEXT: vzeroupper +; AVX512-FASTLANE-NEXT: retq + %cvt = fptoui <4 x half> %a to <4 x i32> + ret <4 x i32> %cvt +} -- GitLab From f82d0187a7e581d4f8f825021dbcb08e8eb37d61 Mon Sep 17 00:00:00 2001 From: David Green Date: Fri, 22 Mar 2024 14:00:21 +0000 Subject: [PATCH 055/306] [AArch64] Add a test to show incorrect latencies into Bundle instructions. NFC --- llvm/test/CodeGen/AArch64/misched-bundle.mir | 195 +++++++++++++++++++ 1 file changed, 195 insertions(+) create mode 100644 llvm/test/CodeGen/AArch64/misched-bundle.mir diff --git a/llvm/test/CodeGen/AArch64/misched-bundle.mir b/llvm/test/CodeGen/AArch64/misched-bundle.mir new file mode 100644 index 000000000000..a947c04a4229 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/misched-bundle.mir @@ -0,0 +1,195 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4 +# RUN: llc -mtriple=aarch64-none-linux-gnu -mcpu=cortex-a510 -run-pass=machine-scheduler -debug-only=machine-scheduler %s -o - 2>&1 | FileCheck %s +# REQUIRES: asserts + +# CHECK: SU(0): renamable $z0 = LD1H renamable $p0, renamable $x1, renamable $x10 :: (load unknown-size, align 1) +# CHECK-NEXT: # preds left : 0 +# CHECK-NEXT: # succs left : 4 +# CHECK-NEXT: # rdefs left : 0 +# CHECK-NEXT: Latency : 3 +# CHECK-NEXT: Depth : 0 +# CHECK-NEXT: Height : 7 +# CHECK-NEXT: Successors: +# CHECK-NEXT: SU(6): Out Latency=1 +# CHECK-NEXT: SU(6): Data Latency=3 Reg=$z0 +# CHECK-NEXT: SU(9): Ord Latency=0 Memory +# CHECK-NEXT: SU(8): Ord Latency=0 Memory +# CHECK-NEXT: Single Issue : false; +# CHECK-NEXT: SU(1): renamable $z1 = LD1H renamable $p0, renamable $x2, renamable $x10 :: (load unknown-size, align 1) +# CHECK-NEXT: # preds left : 0 +# CHECK-NEXT: # succs left : 4 +# CHECK-NEXT: # rdefs left : 0 +# CHECK-NEXT: Latency : 3 +# CHECK-NEXT: Depth : 0 +# CHECK-NEXT: Height : 7 +# CHECK-NEXT: Successors: +# CHECK-NEXT: SU(7): Out Latency=1 +# CHECK-NEXT: SU(6): Data Latency=3 Reg=$z1 +# CHECK-NEXT: SU(9): Ord Latency=0 Memory +# CHECK-NEXT: SU(8): Ord Latency=0 Memory +# CHECK-NEXT: Single Issue : false; +# CHECK-NEXT: SU(2): renamable $z2 = LD1H renamable $p0, renamable $x0, renamable $x10 :: (load unknown-size, align 1) +# CHECK-NEXT: # preds left : 0 +# CHECK-NEXT: # succs left : 3 +# CHECK-NEXT: # rdefs left : 0 +# CHECK-NEXT: Latency : 3 +# CHECK-NEXT: Depth : 0 +# CHECK-NEXT: Height : 7 +# CHECK-NEXT: Successors: +# CHECK-NEXT: SU(6): Data Latency=3 Reg=$z2 +# CHECK-NEXT: SU(9): Ord Latency=0 Memory +# CHECK-NEXT: SU(8): Ord Latency=0 Memory +# CHECK-NEXT: Single Issue : false; +# CHECK-NEXT: SU(3): renamable $z3 = LD1H renamable $p0, renamable $x11, renamable $x10 :: (load unknown-size, align 1) +# CHECK-NEXT: # preds left : 0 +# CHECK-NEXT: # succs left : 3 +# CHECK-NEXT: # rdefs left : 0 +# CHECK-NEXT: Latency : 3 +# CHECK-NEXT: Depth : 0 +# CHECK-NEXT: Height : 0 +# CHECK-NEXT: Successors: +# CHECK-NEXT: SU(7): Data Latency=0 Reg=$z3 +# CHECK-NEXT: SU(9): Ord Latency=0 Memory +# CHECK-NEXT: SU(8): Ord Latency=0 Memory +# CHECK-NEXT: Single Issue : false; +# CHECK-NEXT: SU(4): renamable $z4 = LD1H renamable $p0, renamable $x12, renamable $x10 :: (load unknown-size, align 1) +# CHECK-NEXT: # preds left : 0 +# CHECK-NEXT: # succs left : 3 +# CHECK-NEXT: # rdefs left : 0 +# CHECK-NEXT: Latency : 3 +# CHECK-NEXT: Depth : 0 +# CHECK-NEXT: Height : 0 +# CHECK-NEXT: Successors: +# CHECK-NEXT: SU(7): Data Latency=0 Reg=$z4 +# CHECK-NEXT: SU(9): Ord Latency=0 Memory +# CHECK-NEXT: SU(8): Ord Latency=0 Memory +# CHECK-NEXT: Single Issue : false; +# CHECK-NEXT: SU(5): renamable $z5 = LD1H renamable $p0, renamable $x13, renamable $x10 :: (load unknown-size, align 1) +# CHECK-NEXT: # preds left : 0 +# CHECK-NEXT: # succs left : 3 +# CHECK-NEXT: # rdefs left : 0 +# CHECK-NEXT: Latency : 3 +# CHECK-NEXT: Depth : 0 +# CHECK-NEXT: Height : 0 +# CHECK-NEXT: Successors: +# CHECK-NEXT: SU(7): Data Latency=0 Reg=$z5 +# CHECK-NEXT: SU(9): Ord Latency=0 Memory +# CHECK-NEXT: SU(8): Ord Latency=0 Memory +# CHECK-NEXT: Single Issue : false; +# CHECK-NEXT: SU(6): $z0 = FMAD_ZPmZZ_H renamable $p0, killed $z0(tied-def 0), killed renamable $z1, killed renamable $z2 +# CHECK-NEXT: # preds left : 4 +# CHECK-NEXT: # succs left : 2 +# CHECK-NEXT: # rdefs left : 0 +# CHECK-NEXT: Latency : 4 +# CHECK-NEXT: Depth : 3 +# CHECK-NEXT: Height : 4 +# CHECK-NEXT: Predecessors: +# CHECK-NEXT: SU(2): Data Latency=3 Reg=$z2 +# CHECK-NEXT: SU(1): Data Latency=3 Reg=$z1 +# CHECK-NEXT: SU(0): Out Latency=1 +# CHECK-NEXT: SU(0): Data Latency=3 Reg=$z0 +# CHECK-NEXT: Successors: +# CHECK-NEXT: SU(8): Data Latency=4 Reg=$z0 +# CHECK-NEXT: SU(7): Anti Latency=0 +# CHECK-NEXT: Single Issue : false; +# CHECK-NEXT: SU(7): BUNDLE implicit-def $z1, implicit-def $q1, implicit-def $d1, implicit-def $s1, implicit-def $h1, implicit-def $b1, implicit $z5, implicit $p0, implicit killed $z4, implicit killed $z3 +# CHECK-NEXT: # preds left : 5 +# CHECK-NEXT: # succs left : 1 +# CHECK-NEXT: # rdefs left : 0 +# CHECK-NEXT: Latency : 1 +# CHECK-NEXT: Depth : 3 +# CHECK-NEXT: Height : 0 +# CHECK-NEXT: Predecessors: +# CHECK-NEXT: SU(6): Anti Latency=0 +# CHECK-NEXT: SU(5): Data Latency=0 Reg=$z5 +# CHECK-NEXT: SU(4): Data Latency=0 Reg=$z4 +# CHECK-NEXT: SU(3): Data Latency=0 Reg=$z3 +# CHECK-NEXT: SU(1): Out Latency=1 +# CHECK-NEXT: Successors: +# CHECK-NEXT: SU(9): Data Latency=0 Reg=$z1 +# CHECK-NEXT: Single Issue : false; +# CHECK-NEXT: SU(8): ST1H killed renamable $z0, renamable $p0, renamable $x0, renamable $x10 :: (store unknown-size, align 1) +# CHECK-NEXT: # preds left : 7 +# CHECK-NEXT: # succs left : 1 +# CHECK-NEXT: # rdefs left : 0 +# CHECK-NEXT: Latency : 1 +# CHECK-NEXT: Depth : 7 +# CHECK-NEXT: Height : 0 +# CHECK-NEXT: Predecessors: +# CHECK-NEXT: SU(6): Data Latency=4 Reg=$z0 +# CHECK-NEXT: SU(5): Ord Latency=0 Memory +# CHECK-NEXT: SU(4): Ord Latency=0 Memory +# CHECK-NEXT: SU(3): Ord Latency=0 Memory +# CHECK-NEXT: SU(2): Ord Latency=0 Memory +# CHECK-NEXT: SU(1): Ord Latency=0 Memory +# CHECK-NEXT: SU(0): Ord Latency=0 Memory +# CHECK-NEXT: Successors: +# CHECK-NEXT: SU(9): Ord Latency=0 Memory +# CHECK-NEXT: Single Issue : false; +# CHECK-NEXT: SU(9): ST1H killed renamable $z1, renamable $p0, renamable $x13, renamable $x10 :: (store unknown-size, align 1) +# CHECK-NEXT: # preds left : 8 +# CHECK-NEXT: # succs left : 0 +# CHECK-NEXT: # rdefs left : 0 +# CHECK-NEXT: Latency : 1 +# CHECK-NEXT: Depth : 7 +# CHECK-NEXT: Height : 0 +# CHECK-NEXT: Predecessors: +# CHECK-NEXT: SU(8): Ord Latency=0 Memory +# CHECK-NEXT: SU(7): Data Latency=0 Reg=$z1 +# CHECK-NEXT: SU(5): Ord Latency=0 Memory +# CHECK-NEXT: SU(4): Ord Latency=0 Memory +# CHECK-NEXT: SU(3): Ord Latency=0 Memory +# CHECK-NEXT: SU(2): Ord Latency=0 Memory +# CHECK-NEXT: SU(1): Ord Latency=0 Memory +# CHECK-NEXT: SU(0): Ord Latency=0 Memory +# CHECK-NEXT: Single Issue : false; +# CHECK-NEXT: ExitSU: RET_ReallyLR +# CHECK-NEXT: # preds left : 0 +# CHECK-NEXT: # succs left : 0 +# CHECK-NEXT: # rdefs left : 0 +# CHECK-NEXT: Latency : 0 +# CHECK-NEXT: Depth : 0 +# CHECK-NEXT: Height : 0 + +--- +name: test +alignment: 4 +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: $p0, $x0, $x1, $x2, $x10, $x11, $x12, $x13 + + ; CHECK-LABEL: name: test + ; CHECK: liveins: $p0, $x0, $x1, $x2, $x10, $x11, $x12, $x13 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: renamable $z0 = LD1H renamable $p0, renamable $x1, renamable $x10 :: (load unknown-size, align 1) + ; CHECK-NEXT: renamable $z1 = LD1H renamable $p0, renamable $x2, renamable $x10 :: (load unknown-size, align 1) + ; CHECK-NEXT: renamable $z2 = LD1H renamable $p0, renamable $x0, renamable $x10 :: (load unknown-size, align 1) + ; CHECK-NEXT: $z0 = FMAD_ZPmZZ_H renamable $p0, killed $z0, renamable $z1, killed renamable $z2 + ; CHECK-NEXT: renamable $z3 = LD1H renamable $p0, renamable $x11, renamable $x10 :: (load unknown-size, align 1) + ; CHECK-NEXT: renamable $z4 = LD1H renamable $p0, renamable $x12, renamable $x10 :: (load unknown-size, align 1) + ; CHECK-NEXT: renamable $z5 = LD1H renamable $p0, renamable $x13, renamable $x10 :: (load unknown-size, align 1) + ; CHECK-NEXT: ST1H killed renamable $z0, renamable $p0, renamable $x0, renamable $x10 :: (store unknown-size, align 1) + ; CHECK-NEXT: BUNDLE implicit-def $z1, implicit-def $q1, implicit-def $d1, implicit-def $s1, implicit-def $h1, implicit-def $b1, implicit $z5, implicit $p0, implicit $z4, implicit $z3 { + ; CHECK-NEXT: $z1 = MOVPRFX_ZZ $z5 + ; CHECK-NEXT: $z1 = FMLA_ZPmZZ_H renamable $p0, internal $z1, renamable $z4, renamable $z3 + ; CHECK-NEXT: } + ; CHECK-NEXT: ST1H renamable $z1, renamable $p0, renamable $x13, renamable $x10 :: (store unknown-size, align 1) + ; CHECK-NEXT: RET_ReallyLR + + renamable $z0 = LD1H renamable $p0, renamable $x1, renamable $x10 :: (load unknown-size) + renamable $z1 = LD1H renamable $p0, renamable $x2, renamable $x10 :: (load unknown-size) + renamable $z2 = LD1H renamable $p0, renamable $x0, renamable $x10 :: (load unknown-size) + renamable $z3 = LD1H renamable $p0, renamable $x11, renamable $x10 :: (load unknown-size) + renamable $z4 = LD1H renamable $p0, renamable $x12, renamable $x10 :: (load unknown-size) + renamable $z5 = LD1H renamable $p0, renamable $x13, renamable $x10 :: (load unknown-size) + $z0 = FMAD_ZPmZZ_H renamable $p0, killed $z0, killed renamable $z1, killed renamable $z2 + BUNDLE implicit-def $z1, implicit-def $q1, implicit-def $d1, implicit-def $s1, implicit-def $h1, implicit-def $b1, implicit $z5, implicit $p0, implicit killed $z4, implicit killed $z3 { + $z1 = MOVPRFX_ZZ $z5 + $z1 = FMLA_ZPmZZ_H renamable $p0, internal killed $z1, killed renamable $z4, killed renamable $z3 + } + ST1H killed renamable $z0, renamable $p0, renamable $x0, renamable $x10 :: (store unknown-size) + ST1H killed renamable $z1, renamable $p0, renamable $x13, renamable $x10 :: (store unknown-size) + RET_ReallyLR + +... -- GitLab From e54af608160350baa7ae1b8069f916eb625beadd Mon Sep 17 00:00:00 2001 From: Hirofumi Nakamura Date: Fri, 22 Mar 2024 23:11:36 +0900 Subject: [PATCH 056/306] [clang-format] Added AlignConsecutiveTableGenBreakingDAGArgColons option. (#86150) The option to specify the style of alignment of the colons inside TableGen's DAGArg. --- clang/docs/ClangFormatStyleOptions.rst | 145 ++++++++++++++++++ clang/include/clang/Format/Format.h | 17 ++ clang/lib/Format/Format.cpp | 3 + clang/lib/Format/FormatToken.h | 1 + clang/lib/Format/TokenAnnotator.cpp | 18 ++- clang/lib/Format/WhitespaceManager.cpp | 6 + clang/lib/Format/WhitespaceManager.h | 3 + clang/unittests/Format/FormatTestTableGen.cpp | 32 ++++ clang/unittests/Format/TokenAnnotatorTest.cpp | 16 ++ 9 files changed, 236 insertions(+), 5 deletions(-) diff --git a/clang/docs/ClangFormatStyleOptions.rst b/clang/docs/ClangFormatStyleOptions.rst index be021dfc5c08..2ee36f24d7ce 100644 --- a/clang/docs/ClangFormatStyleOptions.rst +++ b/clang/docs/ClangFormatStyleOptions.rst @@ -955,6 +955,151 @@ the configuration (without a prefix: ``Auto``). } +.. _AlignConsecutiveTableGenBreakingDAGArgColons: + +**AlignConsecutiveTableGenBreakingDAGArgColons** (``AlignConsecutiveStyle``) :versionbadge:`clang-format 19` :ref:`¶ ` + Style of aligning consecutive TableGen DAGArg operator colons. + If enabled, align the colon inside DAGArg which have line break inside. + This works only when TableGenBreakInsideDAGArg is BreakElements or + BreakAll and the DAGArg is not excepted by + TableGenBreakingDAGArgOperators's effect. + + .. code-block:: c++ + + let dagarg = (ins + a :$src1, + aa :$src2, + aaa:$src3 + ) + + Nested configuration flags: + + Alignment options. + + They can also be read as a whole for compatibility. The choices are: + - None + - Consecutive + - AcrossEmptyLines + - AcrossComments + - AcrossEmptyLinesAndComments + + For example, to align across empty lines and not across comments, either + of these work. + + .. code-block:: c++ + + AlignConsecutiveTableGenBreakingDAGArgColons: AcrossEmptyLines + + AlignConsecutiveTableGenBreakingDAGArgColons: + Enabled: true + AcrossEmptyLines: true + AcrossComments: false + + * ``bool Enabled`` Whether aligning is enabled. + + .. code-block:: c++ + + #define SHORT_NAME 42 + #define LONGER_NAME 0x007f + #define EVEN_LONGER_NAME (2) + #define foo(x) (x * x) + #define bar(y, z) (y + z) + + int a = 1; + int somelongname = 2; + double c = 3; + + int aaaa : 1; + int b : 12; + int ccc : 8; + + int aaaa = 12; + float b = 23; + std::string ccc; + + * ``bool AcrossEmptyLines`` Whether to align across empty lines. + + .. code-block:: c++ + + true: + int a = 1; + int somelongname = 2; + double c = 3; + + int d = 3; + + false: + int a = 1; + int somelongname = 2; + double c = 3; + + int d = 3; + + * ``bool AcrossComments`` Whether to align across comments. + + .. code-block:: c++ + + true: + int d = 3; + /* A comment. */ + double e = 4; + + false: + int d = 3; + /* A comment. */ + double e = 4; + + * ``bool AlignCompound`` Only for ``AlignConsecutiveAssignments``. Whether compound assignments + like ``+=`` are aligned along with ``=``. + + .. code-block:: c++ + + true: + a &= 2; + bbb = 2; + + false: + a &= 2; + bbb = 2; + + * ``bool AlignFunctionPointers`` Only for ``AlignConsecutiveDeclarations``. Whether function pointers are + aligned. + + .. code-block:: c++ + + true: + unsigned i; + int &r; + int *p; + int (*f)(); + + false: + unsigned i; + int &r; + int *p; + int (*f)(); + + * ``bool PadOperators`` Only for ``AlignConsecutiveAssignments``. Whether short assignment + operators are left-padded to the same length as long ones in order to + put all assignment operators to the right of the left hand side. + + .. code-block:: c++ + + true: + a >>= 2; + bbb = 2; + + a = 2; + bbb >>= 2; + + false: + a >>= 2; + bbb = 2; + + a = 2; + bbb >>= 2; + + .. _AlignConsecutiveTableGenCondOperatorColons: **AlignConsecutiveTableGenCondOperatorColons** (``AlignConsecutiveStyle``) :versionbadge:`clang-format 19` :ref:`¶ ` diff --git a/clang/include/clang/Format/Format.h b/clang/include/clang/Format/Format.h index 7ad2579bf777..0720c8283cd7 100644 --- a/clang/include/clang/Format/Format.h +++ b/clang/include/clang/Format/Format.h @@ -414,6 +414,21 @@ struct FormatStyle { /// \version 17 ShortCaseStatementsAlignmentStyle AlignConsecutiveShortCaseStatements; + /// Style of aligning consecutive TableGen DAGArg operator colons. + /// If enabled, align the colon inside DAGArg which have line break inside. + /// This works only when TableGenBreakInsideDAGArg is BreakElements or + /// BreakAll and the DAGArg is not excepted by + /// TableGenBreakingDAGArgOperators's effect. + /// \code + /// let dagarg = (ins + /// a :$src1, + /// aa :$src2, + /// aaa:$src3 + /// ) + /// \endcode + /// \version 19 + AlignConsecutiveStyle AlignConsecutiveTableGenBreakingDAGArgColons; + /// Style of aligning consecutive TableGen cond operator colons. /// Align the colons of cases inside !cond operators. /// \code @@ -4879,6 +4894,8 @@ struct FormatStyle { AlignConsecutiveMacros == R.AlignConsecutiveMacros && AlignConsecutiveShortCaseStatements == R.AlignConsecutiveShortCaseStatements && + AlignConsecutiveTableGenBreakingDAGArgColons == + R.AlignConsecutiveTableGenBreakingDAGArgColons && AlignConsecutiveTableGenCondOperatorColons == R.AlignConsecutiveTableGenCondOperatorColons && AlignConsecutiveTableGenDefinitionColons == diff --git a/clang/lib/Format/Format.cpp b/clang/lib/Format/Format.cpp index 63ec3a88978d..46ed5baaeace 100644 --- a/clang/lib/Format/Format.cpp +++ b/clang/lib/Format/Format.cpp @@ -895,6 +895,8 @@ template <> struct MappingTraits { IO.mapOptional("AlignConsecutiveMacros", Style.AlignConsecutiveMacros); IO.mapOptional("AlignConsecutiveShortCaseStatements", Style.AlignConsecutiveShortCaseStatements); + IO.mapOptional("AlignConsecutiveTableGenBreakingDAGArgColons", + Style.AlignConsecutiveTableGenBreakingDAGArgColons); IO.mapOptional("AlignConsecutiveTableGenCondOperatorColons", Style.AlignConsecutiveTableGenCondOperatorColons); IO.mapOptional("AlignConsecutiveTableGenDefinitionColons", @@ -1408,6 +1410,7 @@ FormatStyle getLLVMStyle(FormatStyle::LanguageKind Language) { LLVMStyle.AlignConsecutiveDeclarations = {}; LLVMStyle.AlignConsecutiveMacros = {}; LLVMStyle.AlignConsecutiveShortCaseStatements = {}; + LLVMStyle.AlignConsecutiveTableGenBreakingDAGArgColons = {}; LLVMStyle.AlignConsecutiveTableGenCondOperatorColons = {}; LLVMStyle.AlignConsecutiveTableGenDefinitionColons = {}; LLVMStyle.AlignEscapedNewlines = FormatStyle::ENAS_Right; diff --git a/clang/lib/Format/FormatToken.h b/clang/lib/Format/FormatToken.h index 06f567059c35..2ddcd5259446 100644 --- a/clang/lib/Format/FormatToken.h +++ b/clang/lib/Format/FormatToken.h @@ -152,6 +152,7 @@ namespace format { TYPE(TableGenCondOperatorComma) \ TYPE(TableGenDAGArgCloser) \ TYPE(TableGenDAGArgListColon) \ + TYPE(TableGenDAGArgListColonToAlign) \ TYPE(TableGenDAGArgListComma) \ TYPE(TableGenDAGArgListCommaToBreak) \ TYPE(TableGenDAGArgOpener) \ diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp index 94d2266555f6..a5cafcbacaa5 100644 --- a/clang/lib/Format/TokenAnnotator.cpp +++ b/clang/lib/Format/TokenAnnotator.cpp @@ -975,12 +975,15 @@ private: // DagArg ::= Value [":" TokVarName] | TokVarName // Appears as a part of SimpleValue6. - bool parseTableGenDAGArg() { + bool parseTableGenDAGArg(bool AlignColon = false) { if (tryToParseTableGenTokVar()) return true; if (parseTableGenValue()) { if (CurrentToken && CurrentToken->is(tok::colon)) { - CurrentToken->setType(TT_TableGenDAGArgListColon); + if (AlignColon) + CurrentToken->setType(TT_TableGenDAGArgListColonToAlign); + else + CurrentToken->setType(TT_TableGenDAGArgListColon); skipToNextNonComment(); return tryToParseTableGenTokVar(); } @@ -1051,8 +1054,11 @@ private: skipToNextNonComment(); return true; } - if (!parseTableGenDAGArg()) + if (!parseTableGenDAGArg( + BreakInside && + Style.AlignConsecutiveTableGenBreakingDAGArgColons.Enabled)) { return false; + } FirstDAGArgListElm = false; } return false; @@ -5130,8 +5136,10 @@ bool TokenAnnotator::spaceRequiredBefore(const AnnotatedLine &Line, if (Left.is(tok::r_brace) && Right.is(tok::r_square)) return true; // Do not insert around colon in DAGArg and cond operator. - if (Right.is(TT_TableGenDAGArgListColon) || - Left.is(TT_TableGenDAGArgListColon)) { + if (Right.isOneOf(TT_TableGenDAGArgListColon, + TT_TableGenDAGArgListColonToAlign) || + Left.isOneOf(TT_TableGenDAGArgListColon, + TT_TableGenDAGArgListColonToAlign)) { return false; } if (Right.is(TT_TableGenCondOperatorColon)) diff --git a/clang/lib/Format/WhitespaceManager.cpp b/clang/lib/Format/WhitespaceManager.cpp index 753be25bfd67..fef85abf79a3 100644 --- a/clang/lib/Format/WhitespaceManager.cpp +++ b/clang/lib/Format/WhitespaceManager.cpp @@ -112,6 +112,7 @@ const tooling::Replacements &WhitespaceManager::generateReplacements() { alignConsecutiveBitFields(); alignConsecutiveAssignments(); if (Style.isTableGen()) { + alignConsecutiveTableGenBreakingDAGArgColons(); alignConsecutiveTableGenCondOperatorColons(); alignConsecutiveTableGenDefinitions(); } @@ -981,6 +982,11 @@ void WhitespaceManager::alignConsecutiveShortCaseStatements() { Changes); } +void WhitespaceManager::alignConsecutiveTableGenBreakingDAGArgColons() { + alignConsecutiveColons(Style.AlignConsecutiveTableGenBreakingDAGArgColons, + TT_TableGenDAGArgListColonToAlign); +} + void WhitespaceManager::alignConsecutiveTableGenCondOperatorColons() { alignConsecutiveColons(Style.AlignConsecutiveTableGenCondOperatorColons, TT_TableGenCondOperatorColon); diff --git a/clang/lib/Format/WhitespaceManager.h b/clang/lib/Format/WhitespaceManager.h index 0ebc6cf8377c..98cf4a260cc4 100644 --- a/clang/lib/Format/WhitespaceManager.h +++ b/clang/lib/Format/WhitespaceManager.h @@ -235,6 +235,9 @@ private: /// Align consecutive short case statements over all \c Changes. void alignConsecutiveShortCaseStatements(); + /// Align consecutive TableGen DAGArg colon over all \c Changes. + void alignConsecutiveTableGenBreakingDAGArgColons(); + /// Align consecutive TableGen cond operator colon over all \c Changes. void alignConsecutiveTableGenCondOperatorColons(); diff --git a/clang/unittests/Format/FormatTestTableGen.cpp b/clang/unittests/Format/FormatTestTableGen.cpp index c96866f0840f..8ca6bf97e5a6 100644 --- a/clang/unittests/Format/FormatTestTableGen.cpp +++ b/clang/unittests/Format/FormatTestTableGen.cpp @@ -411,6 +411,38 @@ TEST_F(FormatTestTableGen, DAGArgBreakAll) { Style); } +TEST_F(FormatTestTableGen, DAGArgAlignment) { + FormatStyle Style = getGoogleStyle(FormatStyle::LK_TableGen); + Style.ColumnLimit = 60; + Style.TableGenBreakInsideDAGArg = FormatStyle::DAS_BreakAll; + Style.TableGenBreakingDAGArgOperators = {"ins", "outs"}; + verifyFormat("def Def : Parent {\n" + " let dagarg = (ins\n" + " a:$src1,\n" + " aa:$src2,\n" + " aaa:$src3\n" + " )\n" + "}\n", + Style); + verifyFormat("def Def : Parent {\n" + " let dagarg = (not a:$src1, aa:$src2, aaa:$src2)\n" + "}\n", + Style); + Style.AlignConsecutiveTableGenBreakingDAGArgColons.Enabled = true; + verifyFormat("def Def : Parent {\n" + " let dagarg = (ins\n" + " a :$src1,\n" + " aa :$src2,\n" + " aaa:$src3\n" + " )\n" + "}\n", + Style); + verifyFormat("def Def : Parent {\n" + " let dagarg = (not a:$src1, aa:$src2, aaa:$src2)\n" + "}\n", + Style); +} + TEST_F(FormatTestTableGen, CondOperatorAlignment) { FormatStyle Style = getGoogleStyle(FormatStyle::LK_TableGen); Style.ColumnLimit = 60; diff --git a/clang/unittests/Format/TokenAnnotatorTest.cpp b/clang/unittests/Format/TokenAnnotatorTest.cpp index 1aa855b34198..2539d3d76ef0 100644 --- a/clang/unittests/Format/TokenAnnotatorTest.cpp +++ b/clang/unittests/Format/TokenAnnotatorTest.cpp @@ -2424,6 +2424,22 @@ TEST_F(TokenAnnotatorTest, UnderstandTableGenTokens) { EXPECT_TOKEN(Tokens[1], tok::identifier, TT_Unknown); // other EXPECT_TOKEN(Tokens[5], tok::comma, TT_TableGenDAGArgListComma); EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_TableGenDAGArgCloser); + + // If TableGenBreakingDAGArgOperators is enabled, it uses + // TT_TableGenDAGArgListColonToAlign to annotate the colon to align. + Style.AlignConsecutiveTableGenBreakingDAGArgColons.Enabled = true; + Tokens = AnnotateValue("(ins type1:$src1, type2:$src2)"); + ASSERT_EQ(Tokens.size(), 10u) << Tokens; + EXPECT_TOKEN(Tokens[1], tok::identifier, + TT_TableGenDAGArgOperatorToBreak); // ins + EXPECT_TOKEN(Tokens[3], tok::colon, TT_TableGenDAGArgListColonToAlign); + EXPECT_TOKEN(Tokens[7], tok::colon, TT_TableGenDAGArgListColonToAlign); + + Tokens = AnnotateValue("(other type1:$src1, type2:$src2)"); + ASSERT_EQ(Tokens.size(), 10u) << Tokens; + EXPECT_TOKEN(Tokens[1], tok::identifier, TT_Unknown); // other + EXPECT_TOKEN(Tokens[3], tok::colon, TT_TableGenDAGArgListColon); + EXPECT_TOKEN(Tokens[7], tok::colon, TT_TableGenDAGArgListColon); } TEST_F(TokenAnnotatorTest, UnderstandConstructors) { -- GitLab From 46b853a82ce64e5213f8dfa2c12c6e6a950018a0 Mon Sep 17 00:00:00 2001 From: Billy Laws Date: Fri, 22 Mar 2024 14:17:06 +0000 Subject: [PATCH 057/306] [MC][COFF][AArch64] Treat ARM64EC/X as ARM64 for relocations (#86019) Since ARM64EC/X objects use regular ARM64 relocations, any special handling must be done for them too. --- llvm/lib/MC/WinCOFFObjectWriter.cpp | 4 ++-- llvm/test/MC/AArch64/coff-relocations.s | 12 +++++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/llvm/lib/MC/WinCOFFObjectWriter.cpp b/llvm/lib/MC/WinCOFFObjectWriter.cpp index f265fafa59e7..3c9ff71b6b06 100644 --- a/llvm/lib/MC/WinCOFFObjectWriter.cpp +++ b/llvm/lib/MC/WinCOFFObjectWriter.cpp @@ -266,7 +266,7 @@ WinCOFFWriter::WinCOFFWriter(WinCOFFObjectWriter &OWriter, // limited range for the immediate offset (+/- 1 MB); create extra offset // label symbols with regular intervals to allow referencing a // non-temporary symbol that is close enough. - UseOffsetLabels = Header.Machine == COFF::IMAGE_FILE_MACHINE_ARM64; + UseOffsetLabels = COFF::isAnyArm64(Header.Machine); } COFFSymbol *WinCOFFWriter::createSymbol(StringRef Name) { @@ -954,7 +954,7 @@ void WinCOFFWriter::recordRelocation(MCAssembler &Asm, Reloc.Data.Type == COFF::IMAGE_REL_I386_REL32) || (Header.Machine == COFF::IMAGE_FILE_MACHINE_ARMNT && Reloc.Data.Type == COFF::IMAGE_REL_ARM_REL32) || - (Header.Machine == COFF::IMAGE_FILE_MACHINE_ARM64 && + (COFF::isAnyArm64(Header.Machine) && Reloc.Data.Type == COFF::IMAGE_REL_ARM64_REL32)) FixedValue += 4; diff --git a/llvm/test/MC/AArch64/coff-relocations.s b/llvm/test/MC/AArch64/coff-relocations.s index fb67a21992c6..2370fd9fb436 100644 --- a/llvm/test/MC/AArch64/coff-relocations.s +++ b/llvm/test/MC/AArch64/coff-relocations.s @@ -1,7 +1,11 @@ // RUN: llvm-mc -triple aarch64-windows -filetype obj -o %t.obj %s -// RUN: llvm-readobj -r %t.obj | FileCheck %s +// RUN: llvm-mc -triple arm64ec-windows -filetype obj -o %t-ec.obj %s +// RUN: llvm-readobj -r %t.obj | FileCheck %s --check-prefixes=CHECK,CHECK-ARM64 +// RUN: llvm-readobj -r %t-ec.obj | FileCheck %s --check-prefixes=CHECK,CHECK-ARM64EC // RUN: llvm-objdump --no-print-imm-hex -d %t.obj | FileCheck %s --check-prefix=DISASM +// RUN: llvm-objdump --no-print-imm-hex -d %t-ec.obj | FileCheck %s --check-prefix=DISASM // RUN: llvm-objdump -s %t.obj | FileCheck %s --check-prefix=DATA +// RUN: llvm-objdump -s %t-ec.obj | FileCheck %s --check-prefix=DATA // IMAGE_REL_ARM64_ADDR32 .Linfo_foo: @@ -71,8 +75,10 @@ tbz x0, #0, target // IMAGE_REL_ARM64_REL32 because IMAGE_REL_ARM64_REL64 does not exist. .xword .Linfo_foo - .Ltable -// CHECK: Format: COFF-ARM64 -// CHECK: Arch: aarch64 +// CHECK-ARM64: Format: COFF-ARM64 +// CHECK-ARM64EC: Format: COFF-ARM64EC +// CHECK-ARM64: Arch: aarch64 +// CHECK-ARM64EC: Arch: aarch64 // CHECK: AddressSize: 64bit // CHECK: Relocations [ // CHECK: Section (1) .text { -- GitLab From d231e3b10ead90e4360f7ceb88e4bca9d42d7d04 Mon Sep 17 00:00:00 2001 From: Aaron Ballman Date: Fri, 22 Mar 2024 10:16:27 -0400 Subject: [PATCH 058/306] [C11] Add test & update status of N1282 and DR087 Our existing diagnostics for catching unsequenced modifications handles test coverage for N1282, which is correcting the standard based on the resolution of DR087. --- clang/test/C/C11/n1282.c | 20 ++++++++++++++++++++ clang/test/C/drs/dr0xx.c | 4 ++++ clang/www/c_dr_status.html | 2 +- clang/www/c_status.html | 2 +- 4 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 clang/test/C/C11/n1282.c diff --git a/clang/test/C/C11/n1282.c b/clang/test/C/C11/n1282.c new file mode 100644 index 000000000000..ed952790c883 --- /dev/null +++ b/clang/test/C/C11/n1282.c @@ -0,0 +1,20 @@ +// RUN: %clang_cc1 -verify -Wunsequenced -Wno-unused-value %s + +/* WG14 N1282: Yes + * Clarification of Expressions + */ + +int g; + +int f(int i) { + g = i; + return 0; +} + +int main(void) { + int x; + x = (10, g = 1, 20) + (30, g = 2, 40); /* Line A */ // expected-warning {{multiple unsequenced modifications to 'g'}} + x = (10, f(1), 20) + (30, f(2), 40); /* Line B */ + x = (g = 1) + (g = 2); /* Line C */ // expected-warning {{multiple unsequenced modifications to 'g'}} + return 0; +} diff --git a/clang/test/C/drs/dr0xx.c b/clang/test/C/drs/dr0xx.c index c93cfb63d604..36de32a93da9 100644 --- a/clang/test/C/drs/dr0xx.c +++ b/clang/test/C/drs/dr0xx.c @@ -73,6 +73,10 @@ * WG14 DR085: yes * Returning from main * + * WG14 DR087: yes + * Order of evaluation + * Note: this DR is covered by C/C11/n1282.c + * * WG14 DR086: yes * Object-like macros in system headers * diff --git a/clang/www/c_dr_status.html b/clang/www/c_dr_status.html index fa2ceb1be58b..ed45123ffd0e 100644 --- a/clang/www/c_dr_status.html +++ b/clang/www/c_dr_status.html @@ -577,7 +577,7 @@ conformance.

87 NAD Order of evaluation - Unknown + Yes 88 diff --git a/clang/www/c_status.html b/clang/www/c_status.html index b1f5ab4cbc4f..0069da74cbd5 100644 --- a/clang/www/c_status.html +++ b/clang/www/c_status.html @@ -401,7 +401,7 @@ conformance.

Clarification of expressions N1282 - Unknown + Yes Extending the lifetime of temporary objects (factored approach) -- GitLab From 8612fa0d84c730a753d04de012a8372ba5a10677 Mon Sep 17 00:00:00 2001 From: agozillon Date: Fri, 22 Mar 2024 15:32:39 +0100 Subject: [PATCH 059/306] [MLIR][OpenMP] Refactor bounds offsetting and fix to apply to all directives (#84349) This PR refactors bounds offsetting by combining the two differing implementations (one applying to initial derived type member map implementation for descriptors and the other for regular arrays, effectively allocatable array vs regular array in fortran) now that it's a little simpler to do. The PR also moves the utilization of createAlteredByCaptureMap into genMapInfoOp, where it will be correctly applied to all MapInfoData, appropriately offsetting and altering Pointer data set in the kernel argument structure on the host. This primarily means bounds offsets will now correctly apply to enter/exit/update map clauses as opposed to just the Target directive that is currently the case. A few fortran runtime tests have been added to verify this new behavior. This PR depends on: https://github.com/llvm/llvm-project/pull/84328 and is an extraction of the larger derived type member map PR stack (so a requirement for it to land). --- .../OpenMP/OpenMPToLLVMIRTranslation.cpp | 374 +++++++++++------- ...target-fortran-allocatable-types-host.mlir | 11 +- mlir/test/Target/LLVMIR/omptarget-llvm.mlir | 25 +- .../fortran/target-map-enter-exit-array-2.f90 | 39 ++ .../target-map-enter-exit-array-bounds.f90 | 44 +++ .../fortran/target-map-enter-exit-scalar.f90 | 33 ++ 6 files changed, 360 insertions(+), 166 deletions(-) create mode 100644 openmp/libomptarget/test/offloading/fortran/target-map-enter-exit-array-2.f90 create mode 100644 openmp/libomptarget/test/offloading/fortran/target-map-enter-exit-array-bounds.f90 create mode 100644 openmp/libomptarget/test/offloading/fortran/target-map-enter-exit-scalar.f90 diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp index 7df33470ea06..646d0ed73084 100644 --- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp @@ -1787,6 +1787,20 @@ getDeclareTargetRefPtrSuffix(LLVM::GlobalOp globalOp, return suffix; } +static bool isDeclareTargetLink(mlir::Value value) { + if (auto addressOfOp = + llvm::dyn_cast_if_present(value.getDefiningOp())) { + auto modOp = addressOfOp->getParentOfType(); + Operation *gOp = modOp.lookupSymbol(addressOfOp.getGlobalName()); + if (auto declareTargetGlobal = + llvm::dyn_cast(gOp)) + if (declareTargetGlobal.getDeclareTargetCaptureClause() == + mlir::omp::DeclareTargetCaptureClause::link) + return true; + } + return false; +} + // Returns the reference pointer generated by the lowering of the declare target // operation in cases where the link clause is used or the to clause is used in // USM mode. @@ -1982,6 +1996,99 @@ void collectMapDataFromMapOperands(MapInfoData &mapData, } } +/// This function calculates the array/pointer offset for map data provided +/// with bounds operations, e.g. when provided something like the following: +/// +/// Fortran +/// map(tofrom: array(2:5, 3:2)) +/// or +/// C++ +/// map(tofrom: array[1:4][2:3]) +/// We must calculate the initial pointer offset to pass across, this function +/// performs this using bounds. +/// +/// NOTE: which while specified in row-major order it currently needs to be +/// flipped for Fortran's column order array allocation and access (as +/// opposed to C++'s row-major, hence the backwards processing where order is +/// important). This is likely important to keep in mind for the future when +/// we incorporate a C++ frontend, both frontends will need to agree on the +/// ordering of generated bounds operations (one may have to flip them) to +/// make the below lowering frontend agnostic. The offload size +/// calcualtion may also have to be adjusted for C++. +std::vector +calculateBoundsOffset(LLVM::ModuleTranslation &moduleTranslation, + llvm::IRBuilderBase &builder, bool isArrayTy, + mlir::OperandRange bounds) { + std::vector idx; + // There's no bounds to calculate an offset from, we can safely + // ignore and return no indices. + if (bounds.empty()) + return idx; + + // If we have an array type, then we have its type so can treat it as a + // normal GEP instruction where the bounds operations are simply indexes + // into the array. We currently do reverse order of the bounds, which + // I believe leans more towards Fortran's column-major in memory. + if (isArrayTy) { + idx.push_back(builder.getInt64(0)); + for (int i = bounds.size() - 1; i >= 0; --i) { + if (auto boundOp = mlir::dyn_cast_if_present( + bounds[i].getDefiningOp())) { + idx.push_back(moduleTranslation.lookupValue(boundOp.getLowerBound())); + } + } + } else { + // If we do not have an array type, but we have bounds, then we're dealing + // with a pointer that's being treated like an array and we have the + // underlying type e.g. an i32, or f64 etc, e.g. a fortran descriptor base + // address (pointer pointing to the actual data) so we must caclulate the + // offset using a single index which the following two loops attempts to + // compute. + + // Calculates the size offset we need to make per row e.g. first row or + // column only needs to be offset by one, but the next would have to be + // the previous row/column offset multiplied by the extent of current row. + // + // For example ([1][10][100]): + // + // - First row/column we move by 1 for each index increment + // - Second row/column we move by 1 (first row/column) * 10 (extent/size of + // current) for 10 for each index increment + // - Third row/column we would move by 10 (second row/column) * + // (extent/size of current) 100 for 1000 for each index increment + std::vector dimensionIndexSizeOffset{builder.getInt64(1)}; + for (size_t i = 1; i < bounds.size(); ++i) { + if (auto boundOp = mlir::dyn_cast_if_present( + bounds[i].getDefiningOp())) { + dimensionIndexSizeOffset.push_back(builder.CreateMul( + moduleTranslation.lookupValue(boundOp.getExtent()), + dimensionIndexSizeOffset[i - 1])); + } + } + + // Now that we have calculated how much we move by per index, we must + // multiply each lower bound offset in indexes by the size offset we + // have calculated in the previous and accumulate the results to get + // our final resulting offset. + for (int i = bounds.size() - 1; i >= 0; --i) { + if (auto boundOp = mlir::dyn_cast_if_present( + bounds[i].getDefiningOp())) { + if (idx.empty()) + idx.emplace_back(builder.CreateMul( + moduleTranslation.lookupValue(boundOp.getLowerBound()), + dimensionIndexSizeOffset[i])); + else + idx.back() = builder.CreateAdd( + idx.back(), builder.CreateMul(moduleTranslation.lookupValue( + boundOp.getLowerBound()), + dimensionIndexSizeOffset[i])); + } + } + } + + return idx; +} + // This creates two insertions into the MapInfosTy data structure for the // "parent" of a set of members, (usually a container e.g. // class/structure/derived type) when subsequent members have also been @@ -2057,6 +2164,27 @@ static llvm::omp::OpenMPOffloadMappingFlags mapParentWithMembers( return memberOfFlag; } +// The intent is to verify if the mapped data being passed is a +// pointer -> pointee that requires special handling in certain cases, +// e.g. applying the OMP_MAP_PTR_AND_OBJ map type. +// +// There may be a better way to verify this, but unfortunately with +// opaque pointers we lose the ability to easily check if something is +// a pointer whilst maintaining access to the underlying type. +static bool checkIfPointerMap(mlir::omp::MapInfoOp mapOp) { + // If we have a varPtrPtr field assigned then the underlying type is a pointer + if (mapOp.getVarPtrPtr()) + return true; + + // If the map data is declare target with a link clause, then it's represented + // as a pointer when we lower it to LLVM-IR even if at the MLIR level it has + // no relation to pointers. + if (isDeclareTargetLink(mapOp.getVarPtr())) + return true; + + return false; +} + // This function is intended to add explicit mappings of members static void processMapMembersWithParent( LLVM::ModuleTranslation &moduleTranslation, llvm::IRBuilderBase &builder, @@ -2083,8 +2211,11 @@ static void processMapMembersWithParent( auto mapFlag = llvm::omp::OpenMPOffloadMappingFlags(memberClause.getMapType().value()); mapFlag &= ~llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM; + mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF; ompBuilder.setCorrectMemberOfFlag(mapFlag, memberOfFlag); - mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ; + if (checkIfPointerMap(memberClause)) + mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ; + combinedInfo.Types.emplace_back(mapFlag); combinedInfo.DevicePointers.emplace_back( llvm::OpenMPIRBuilder::DeviceInfoTy::None); @@ -2092,55 +2223,7 @@ static void processMapMembersWithParent( LLVM::createMappingInformation(memberClause.getLoc(), ompBuilder)); combinedInfo.BasePointers.emplace_back(mapData.BasePointers[memberDataIdx]); - - std::vector idx{builder.getInt64(0)}; - llvm::Value *offsetAddress = nullptr; - if (!memberClause.getBounds().empty()) { - if (mapData.BaseType[memberDataIdx]->isArrayTy()) { - for (int i = memberClause.getBounds().size() - 1; i >= 0; --i) { - if (auto boundOp = mlir::dyn_cast_if_present( - memberClause.getBounds()[i].getDefiningOp())) { - idx.push_back( - moduleTranslation.lookupValue(boundOp.getLowerBound())); - } - } - } else { - std::vector dimensionIndexSizeOffset{ - builder.getInt64(1)}; - for (size_t i = 1; i < memberClause.getBounds().size(); ++i) { - if (auto boundOp = mlir::dyn_cast_if_present( - memberClause.getBounds()[i].getDefiningOp())) { - dimensionIndexSizeOffset.push_back(builder.CreateMul( - moduleTranslation.lookupValue(boundOp.getExtent()), - dimensionIndexSizeOffset[i - 1])); - } - } - - for (int i = memberClause.getBounds().size() - 1; i >= 0; --i) { - if (auto boundOp = mlir::dyn_cast_if_present( - memberClause.getBounds()[i].getDefiningOp())) { - if (!offsetAddress) - offsetAddress = builder.CreateMul( - moduleTranslation.lookupValue(boundOp.getLowerBound()), - dimensionIndexSizeOffset[i]); - else - offsetAddress = builder.CreateAdd( - offsetAddress, - builder.CreateMul( - moduleTranslation.lookupValue(boundOp.getLowerBound()), - dimensionIndexSizeOffset[i])); - } - } - } - } - - llvm::Value *memberIdx = - builder.CreateLoad(builder.getPtrTy(), mapData.Pointers[memberDataIdx]); - memberIdx = builder.CreateInBoundsGEP( - mapData.BaseType[memberDataIdx], memberIdx, - offsetAddress ? std::vector{offsetAddress} : idx, - "member_idx"); - combinedInfo.Pointers.emplace_back(memberIdx); + combinedInfo.Pointers.emplace_back(mapData.Pointers[memberDataIdx]); combinedInfo.Sizes.emplace_back(mapData.Sizes[memberDataIdx]); } } @@ -2158,6 +2241,76 @@ static void processMapWithMembersOf( memberOfParentFlag); } +// This is a variation on Clang's GenerateOpenMPCapturedVars, which +// generates different operation (e.g. load/store) combinations for +// arguments to the kernel, based on map capture kinds which are then +// utilised in the combinedInfo in place of the original Map value. +static void +createAlteredByCaptureMap(MapInfoData &mapData, + LLVM::ModuleTranslation &moduleTranslation, + llvm::IRBuilderBase &builder) { + for (size_t i = 0; i < mapData.MapClause.size(); ++i) { + // if it's declare target, skip it, it's handled seperately. + if (!mapData.IsDeclareTarget[i]) { + auto mapOp = + mlir::dyn_cast_if_present(mapData.MapClause[i]); + mlir::omp::VariableCaptureKind captureKind = + mapOp.getMapCaptureType().value_or( + mlir::omp::VariableCaptureKind::ByRef); + bool isPtrTy = checkIfPointerMap(mapOp); + + // Currently handles array sectioning lowerbound case, but more + // logic may be required in the future. Clang invokes EmitLValue, + // which has specialised logic for special Clang types such as user + // defines, so it is possible we will have to extend this for + // structures or other complex types. As the general idea is that this + // function mimics some of the logic from Clang that we require for + // kernel argument passing from host -> device. + switch (captureKind) { + case mlir::omp::VariableCaptureKind::ByRef: { + llvm::Value *newV = mapData.Pointers[i]; + std::vector offsetIdx = calculateBoundsOffset( + moduleTranslation, builder, mapData.BaseType[i]->isArrayTy(), + mapOp.getBounds()); + if (isPtrTy) + newV = builder.CreateLoad(builder.getPtrTy(), newV); + + if (!offsetIdx.empty()) + newV = builder.CreateInBoundsGEP(mapData.BaseType[i], newV, offsetIdx, + "array_offset"); + mapData.Pointers[i] = newV; + } break; + case mlir::omp::VariableCaptureKind::ByCopy: { + llvm::Type *type = mapData.BaseType[i]; + llvm::Value *newV; + if (mapData.Pointers[i]->getType()->isPointerTy()) + newV = builder.CreateLoad(type, mapData.Pointers[i]); + else + newV = mapData.Pointers[i]; + + if (!isPtrTy) { + auto curInsert = builder.saveIP(); + builder.restoreIP(findAllocaInsertPoint(builder, moduleTranslation)); + auto *memTempAlloc = + builder.CreateAlloca(builder.getPtrTy(), nullptr, ".casted"); + builder.restoreIP(curInsert); + + builder.CreateStore(newV, memTempAlloc); + newV = builder.CreateLoad(builder.getPtrTy(), memTempAlloc); + } + + mapData.Pointers[i] = newV; + mapData.BasePointers[i] = newV; + } break; + case mlir::omp::VariableCaptureKind::This: + case mlir::omp::VariableCaptureKind::VLAType: + mapData.MapClause[i]->emitOpError("Unhandled capture kind"); + break; + } + } + } +} + // Generate all map related information and fill the combinedInfo. static void genMapInfos(llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation, @@ -2167,6 +2320,20 @@ static void genMapInfos(llvm::IRBuilderBase &builder, const SmallVector &devPtrOperands = {}, const SmallVector &devAddrOperands = {}, bool isTargetParams = false) { + // We wish to modify some of the methods in which arguments are + // passed based on their capture type by the target region, this can + // involve generating new loads and stores, which changes the + // MLIR value to LLVM value mapping, however, we only wish to do this + // locally for the current function/target and also avoid altering + // ModuleTranslation, so we remap the base pointer or pointer stored + // in the map infos corresponding MapInfoData, which is later accessed + // by genMapInfos and createTarget to help generate the kernel and + // kernel arg structure. It primarily becomes relevant in cases like + // bycopy, or byref range'd arrays. In the default case, we simply + // pass thee pointer byref as both basePointer and pointer. + if (!moduleTranslation.getOpenMPBuilder()->Config.isTargetDevice()) + createAlteredByCaptureMap(mapData, moduleTranslation, builder); + llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder(); auto fail = [&combinedInfo]() -> void { @@ -2200,19 +2367,20 @@ static void genMapInfos(llvm::IRBuilderBase &builder, continue; } - // Declare Target Mappings are excluded from being marked as - // OMP_MAP_TARGET_PARAM as they are not passed as parameters, they're - // marked with OMP_MAP_PTR_AND_OBJ instead. auto mapFlag = mapData.Types[i]; - if (mapData.IsDeclareTarget[i]) + bool isPtrTy = checkIfPointerMap(mapInfoOp); + if (isPtrTy) mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ; - else if (isTargetParams) + + // Declare Target Mappings are excluded from being marked as + // OMP_MAP_TARGET_PARAM as they are not passed as parameters. + if (isTargetParams && !mapData.IsDeclareTarget[i]) mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM; if (auto mapInfoOp = dyn_cast(mapData.MapClause[i])) if (mapInfoOp.getMapCaptureType().value() == mlir::omp::VariableCaptureKind::ByCopy && - !mapInfoOp.getVarType().isa()) + !isPtrTy) mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_LITERAL; combinedInfo.BasePointers.emplace_back(mapData.BasePointers[i]); @@ -2662,86 +2830,6 @@ createDeviceArgumentAccessor(MapInfoData &mapData, llvm::Argument &arg, return builder.saveIP(); } -// This is a variation on Clang's GenerateOpenMPCapturedVars, which -// generates different operation (e.g. load/store) combinations for -// arguments to the kernel, based on map capture kinds which are then -// utilised in the combinedInfo in place of the original Map value. -static void -createAlteredByCaptureMap(MapInfoData &mapData, - LLVM::ModuleTranslation &moduleTranslation, - llvm::IRBuilderBase &builder) { - for (size_t i = 0; i < mapData.MapClause.size(); ++i) { - // if it's declare target, skip it, it's handled seperately. - if (!mapData.IsDeclareTarget[i]) { - mlir::omp::VariableCaptureKind captureKind = - mlir::omp::VariableCaptureKind::ByRef; - - if (auto mapOp = mlir::dyn_cast_if_present( - mapData.MapClause[i])) { - captureKind = mapOp.getMapCaptureType().value_or( - mlir::omp::VariableCaptureKind::ByRef); - } - - switch (captureKind) { - case mlir::omp::VariableCaptureKind::ByRef: { - // Currently handles array sectioning lowerbound case, but more - // logic may be required in the future. Clang invokes EmitLValue, - // which has specialised logic for special Clang types such as user - // defines, so it is possible we will have to extend this for - // structures or other complex types. As the general idea is that this - // function mimics some of the logic from Clang that we require for - // kernel argument passing from host -> device. - if (auto mapOp = mlir::dyn_cast_if_present( - mapData.MapClause[i])) { - if (!mapOp.getBounds().empty() && mapData.BaseType[i]->isArrayTy()) { - - std::vector idx = - std::vector{builder.getInt64(0)}; - for (int i = mapOp.getBounds().size() - 1; i >= 0; --i) { - if (auto boundOp = - mlir::dyn_cast_if_present( - mapOp.getBounds()[i].getDefiningOp())) { - idx.push_back( - moduleTranslation.lookupValue(boundOp.getLowerBound())); - } - } - - mapData.Pointers[i] = builder.CreateInBoundsGEP( - mapData.BaseType[i], mapData.Pointers[i], idx); - } - } - } break; - case mlir::omp::VariableCaptureKind::ByCopy: { - llvm::Type *type = mapData.BaseType[i]; - llvm::Value *newV; - if (mapData.Pointers[i]->getType()->isPointerTy()) - newV = builder.CreateLoad(type, mapData.Pointers[i]); - else - newV = mapData.Pointers[i]; - - if (!type->isPointerTy()) { - auto curInsert = builder.saveIP(); - builder.restoreIP(findAllocaInsertPoint(builder, moduleTranslation)); - auto *memTempAlloc = - builder.CreateAlloca(builder.getPtrTy(), nullptr, ".casted"); - builder.restoreIP(curInsert); - - builder.CreateStore(newV, memTempAlloc); - newV = builder.CreateLoad(builder.getPtrTy(), memTempAlloc); - } - - mapData.Pointers[i] = newV; - mapData.BasePointers[i] = newV; - } break; - case mlir::omp::VariableCaptureKind::This: - case mlir::omp::VariableCaptureKind::VLAType: - mapData.MapClause[i]->emitOpError("Unhandled capture kind"); - break; - } - } - } -} - static LogicalResult convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation) { @@ -2810,20 +2898,6 @@ convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder, collectMapDataFromMapOperands(mapData, mapOperands, moduleTranslation, dl, builder); - // We wish to modify some of the methods in which kernel arguments are - // passed based on their capture type by the target region, this can - // involve generating new loads and stores, which changes the - // MLIR value to LLVM value mapping, however, we only wish to do this - // locally for the current function/target and also avoid altering - // ModuleTranslation, so we remap the base pointer or pointer stored - // in the map infos corresponding MapInfoData, which is later accessed - // by genMapInfos and createTarget to help generate the kernel and - // kernel arg structure. It primarily becomes relevant in cases like - // bycopy, or byref range'd arrays. In the default case, we simply - // pass thee pointer byref as both basePointer and pointer. - if (!moduleTranslation.getOpenMPBuilder()->Config.isTargetDevice()) - createAlteredByCaptureMap(mapData, moduleTranslation, builder); - llvm::OpenMPIRBuilder::MapInfosTy combinedInfos; auto genMapInfoCB = [&](llvm::OpenMPIRBuilder::InsertPointTy codeGenIP) -> llvm::OpenMPIRBuilder::MapInfosTy & { diff --git a/mlir/test/Target/LLVMIR/omptarget-fortran-allocatable-types-host.mlir b/mlir/test/Target/LLVMIR/omptarget-fortran-allocatable-types-host.mlir index e8c388627a0a..7cb22dbb10b1 100644 --- a/mlir/test/Target/LLVMIR/omptarget-fortran-allocatable-types-host.mlir +++ b/mlir/test/Target/LLVMIR/omptarget-fortran-allocatable-types-host.mlir @@ -26,7 +26,7 @@ module attributes {omp.is_target_device = false} { %14 = llvm.sub %11, %2 : i64 %15 = omp.map.bounds lower_bound(%7 : i64) upper_bound(%14 : i64) extent(%11 : i64) stride(%13 : i64) start_idx(%9 : i64) {stride_in_bytes = true} %16 = llvm.getelementptr %3[0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)> - %17 = omp.map.info var_ptr(%16 : !llvm.ptr, f32) map_clauses(tofrom) capture(ByRef) bounds(%15) -> !llvm.ptr {name = "full_arr"} + %17 = omp.map.info var_ptr(%3 : !llvm.ptr, f32) var_ptr_ptr(%16 : !llvm.ptr) map_clauses(tofrom) capture(ByRef) bounds(%15) -> !llvm.ptr {name = "full_arr"} %18 = omp.map.info var_ptr(%3 : !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)>) map_clauses(tofrom) capture(ByRef) members(%17 : !llvm.ptr) -> !llvm.ptr {name = "full_arr"} %19 = llvm.getelementptr %6[0, 7, %7, 0] : (!llvm.ptr, i64) -> !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)> %20 = llvm.load %19 : !llvm.ptr -> i64 @@ -81,20 +81,19 @@ module attributes {omp.is_target_device = false} { // CHECK: %[[ARR_SECT_SIZE2:.*]] = add i64 %[[ARR_SECT_SIZE3]], 1 // CHECK: %[[ARR_SECT_SIZE1:.*]] = mul i64 1, %[[ARR_SECT_SIZE2]] // CHECK: %[[ARR_SECT_SIZE:.*]] = mul i64 %[[ARR_SECT_SIZE1]], 4 -// CHECK: %[[FULL_ARR_DESC_SIZE:.*]] = sdiv exact i64 sub (i64 ptrtoint (ptr getelementptr inbounds ({ ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr @_QFEfull_arr, i32 1) to i64), i64 ptrtoint (ptr @_QFEfull_arr to i64)), ptrtoint (ptr getelementptr (i8, ptr null, i32 1) to i64) // CHECK: %[[LFULL_ARR:.*]] = load ptr, ptr @_QFEfull_arr, align 8 // CHECK: %[[FULL_ARR_PTR:.*]] = getelementptr inbounds float, ptr %[[LFULL_ARR]], i64 0 -// CHECK: %[[ARR_SECT_DESC_SIZE:.*]] = sdiv exact i64 sub (i64 ptrtoint (ptr getelementptr inbounds ({ ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr @_QFEsect_arr, i32 1) to i64), i64 ptrtoint (ptr @_QFEsect_arr to i64)), ptrtoint (ptr getelementptr (i8, ptr null, i32 1) to i64) // CHECK: %[[ARR_SECT_OFFSET1:.*]] = mul i64 %[[ARR_SECT_OFFSET2]], 1 // CHECK: %[[LARR_SECT:.*]] = load ptr, ptr @_QFEsect_arr, align 8 // CHECK: %[[ARR_SECT_PTR:.*]] = getelementptr inbounds i32, ptr %[[LARR_SECT]], i64 %[[ARR_SECT_OFFSET1]] +// CHECK: %[[SCALAR_PTR_LOAD:.*]] = load ptr, ptr %[[SCALAR_BASE]], align 8 +// CHECK: %[[FULL_ARR_DESC_SIZE:.*]] = sdiv exact i64 sub (i64 ptrtoint (ptr getelementptr inbounds ({ ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr @_QFEfull_arr, i32 1) to i64), i64 ptrtoint (ptr @_QFEfull_arr to i64)), ptrtoint (ptr getelementptr (i8, ptr null, i32 1) to i64) +// CHECK: %[[ARR_SECT_DESC_SIZE:.*]] = sdiv exact i64 sub (i64 ptrtoint (ptr getelementptr inbounds ({ ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr @_QFEsect_arr, i32 1) to i64), i64 ptrtoint (ptr @_QFEsect_arr to i64)), ptrtoint (ptr getelementptr (i8, ptr null, i32 1) to i64) // CHECK: %[[SCALAR_DESC_SZ4:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8 }, ptr %[[SCALAR_ALLOCA]], i32 1 // CHECK: %[[SCALAR_DESC_SZ3:.*]] = ptrtoint ptr %[[SCALAR_DESC_SZ4]] to i64 // CHECK: %[[SCALAR_DESC_SZ2:.*]] = ptrtoint ptr %[[SCALAR_ALLOCA]] to i64 // CHECK: %[[SCALAR_DESC_SZ1:.*]] = sub i64 %[[SCALAR_DESC_SZ3]], %[[SCALAR_DESC_SZ2]] // CHECK: %[[SCALAR_DESC_SZ:.*]] = sdiv exact i64 %[[SCALAR_DESC_SZ1]], ptrtoint (ptr getelementptr (i8, ptr null, i32 1) to i64) -// CHECK: %[[SCALAR_PTR_LOAD:.*]] = load ptr, ptr %[[SCALAR_BASE]], align 8 -// CHECK: %[[SCALAR_PTR:.*]] = getelementptr inbounds float, ptr %[[SCALAR_PTR_LOAD]], i64 0 // CHECK: %[[OFFLOADBASEPTRS:.*]] = getelementptr inbounds [9 x ptr], ptr %.offload_baseptrs, i32 0, i32 0 // CHECK: store ptr @_QFEfull_arr, ptr %[[OFFLOADBASEPTRS]], align 8 @@ -145,4 +144,4 @@ module attributes {omp.is_target_device = false} { // CHECK: %[[OFFLOADBASEPTRS:.*]] = getelementptr inbounds [9 x ptr], ptr %.offload_baseptrs, i32 0, i32 8 // CHECK: store ptr %[[SCALAR_BASE]], ptr %[[OFFLOADBASEPTRS]], align 8 // CHECK: %[[OFFLOADPTRS:.*]] = getelementptr inbounds [9 x ptr], ptr %.offload_ptrs, i32 0, i32 8 -// CHECK: store ptr %[[SCALAR_PTR]], ptr %[[OFFLOADPTRS]], align 8 +// CHECK: store ptr %[[SCALAR_PTR_LOAD]], ptr %[[OFFLOADPTRS]], align 8 diff --git a/mlir/test/Target/LLVMIR/omptarget-llvm.mlir b/mlir/test/Target/LLVMIR/omptarget-llvm.mlir index 4b1d5d58f14e..2f629675442d 100644 --- a/mlir/test/Target/LLVMIR/omptarget-llvm.mlir +++ b/mlir/test/Target/LLVMIR/omptarget-llvm.mlir @@ -66,16 +66,17 @@ llvm.func @_QPopenmp_target_data_region(%0 : !llvm.ptr) { // CHECK: %[[VAL_2:.*]] = alloca [1 x ptr], align 8 // CHECK: br label %[[VAL_3:.*]] // CHECK: entry: ; preds = %[[VAL_4:.*]] +// CHECK: %[[ARR_OFFSET:.*]] = getelementptr inbounds [1024 x i32], ptr %[[ARR_DATA:.*]], i64 0, i64 0 // CHECK: %[[VAL_5:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_0]], i32 0, i32 0 -// CHECK: store ptr %[[VAL_6:.*]], ptr %[[VAL_5]], align 8 +// CHECK: store ptr %[[ARR_DATA]], ptr %[[VAL_5]], align 8 // CHECK: %[[VAL_7:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_1]], i32 0, i32 0 -// CHECK: store ptr %[[VAL_6]], ptr %[[VAL_7]], align 8 +// CHECK: store ptr %[[ARR_OFFSET]], ptr %[[VAL_7]], align 8 // CHECK: %[[VAL_8:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_2]], i64 0, i64 0 // CHECK: store ptr null, ptr %[[VAL_8]], align 8 // CHECK: %[[VAL_9:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_0]], i32 0, i32 0 // CHECK: %[[VAL_10:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_1]], i32 0, i32 0 // CHECK: call void @__tgt_target_data_begin_mapper(ptr @2, i64 -1, i32 1, ptr %[[VAL_9]], ptr %[[VAL_10]], ptr @.offload_sizes, ptr @.offload_maptypes, ptr @.offload_mapnames, ptr null) -// CHECK: %[[VAL_11:.*]] = getelementptr [1024 x i32], ptr %[[VAL_6]], i32 0, i64 0 +// CHECK: %[[VAL_11:.*]] = getelementptr [1024 x i32], ptr %[[ARR_DATA]], i32 0, i64 0 // CHECK: store i32 99, ptr %[[VAL_11]], align 4 // CHECK: %[[VAL_12:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_0]], i32 0, i32 0 // CHECK: %[[VAL_13:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_1]], i32 0, i32 0 @@ -153,16 +154,18 @@ llvm.func @_QPomp_target_enter_exit(%1 : !llvm.ptr, %3 : !llvm.ptr) { // CHECK: entry: ; preds = %[[VAL_12:.*]] // CHECK: br i1 %[[VAL_9]], label %[[VAL_13:.*]], label %[[VAL_14:.*]] // CHECK: omp_if.then: ; preds = %[[VAL_11]] +// CHECK: %[[ARR_OFFSET1:.*]] = getelementptr inbounds [1024 x i32], ptr %[[VAL_16:.*]], i64 0, i64 0 +// CHECK: %[[ARR_OFFSET2:.*]] = getelementptr inbounds [512 x i32], ptr %[[VAL_20:.*]], i64 0, i64 0 // CHECK: %[[VAL_15:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_3]], i32 0, i32 0 // CHECK: store ptr %[[VAL_16:.*]], ptr %[[VAL_15]], align 8 // CHECK: %[[VAL_17:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_4]], i32 0, i32 0 -// CHECK: store ptr %[[VAL_16]], ptr %[[VAL_17]], align 8 +// CHECK: store ptr %[[ARR_OFFSET1]], ptr %[[VAL_17]], align 8 // CHECK: %[[VAL_18:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_5]], i64 0, i64 0 // CHECK: store ptr null, ptr %[[VAL_18]], align 8 // CHECK: %[[VAL_19:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_3]], i32 0, i32 1 // CHECK: store ptr %[[VAL_20:.*]], ptr %[[VAL_19]], align 8 // CHECK: %[[VAL_21:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_4]], i32 0, i32 1 -// CHECK: store ptr %[[VAL_20]], ptr %[[VAL_21]], align 8 +// CHECK: store ptr %[[ARR_OFFSET2]], ptr %[[VAL_21]], align 8 // CHECK: %[[VAL_22:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_5]], i64 0, i64 1 // CHECK: store ptr null, ptr %[[VAL_22]], align 8 // CHECK: %[[VAL_23:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_3]], i32 0, i32 0 @@ -176,26 +179,28 @@ llvm.func @_QPomp_target_enter_exit(%1 : !llvm.ptr, %3 : !llvm.ptr) { // CHECK: %[[VAL_27:.*]] = icmp sgt i32 %[[VAL_26]], 10 // CHECK: %[[VAL_28:.*]] = load i32, ptr %[[VAL_6]], align 4 // CHECK: br i1 %[[VAL_27]], label %[[VAL_29:.*]], label %[[VAL_30:.*]] -// CHECK: omp_if.then1: ; preds = %[[VAL_25]] +// CHECK: omp_if.then2: ; preds = %[[VAL_25]] +// CHECK: %[[ARR_OFFSET3:.*]] = getelementptr inbounds [1024 x i32], ptr %[[VAL_16]], i64 0, i64 0 +// CHECK: %[[ARR_OFFSET4:.*]] = getelementptr inbounds [512 x i32], ptr %[[VAL_20]], i64 0, i64 0 // CHECK: %[[VAL_31:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_0]], i32 0, i32 0 // CHECK: store ptr %[[VAL_16]], ptr %[[VAL_31]], align 8 // CHECK: %[[VAL_32:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_1]], i32 0, i32 0 -// CHECK: store ptr %[[VAL_16]], ptr %[[VAL_32]], align 8 +// CHECK: store ptr %[[ARR_OFFSET3]], ptr %[[VAL_32]], align 8 // CHECK: %[[VAL_33:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_2]], i64 0, i64 0 // CHECK: store ptr null, ptr %[[VAL_33]], align 8 // CHECK: %[[VAL_34:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_0]], i32 0, i32 1 // CHECK: store ptr %[[VAL_20]], ptr %[[VAL_34]], align 8 // CHECK: %[[VAL_35:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_1]], i32 0, i32 1 -// CHECK: store ptr %[[VAL_20]], ptr %[[VAL_35]], align 8 +// CHECK: store ptr %[[ARR_OFFSET4]], ptr %[[VAL_35]], align 8 // CHECK: %[[VAL_36:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_2]], i64 0, i64 1 // CHECK: store ptr null, ptr %[[VAL_36]], align 8 // CHECK: %[[VAL_37:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_0]], i32 0, i32 0 // CHECK: %[[VAL_38:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_1]], i32 0, i32 0 // CHECK: call void @__tgt_target_data_end_mapper(ptr @3, i64 -1, i32 2, ptr %[[VAL_37]], ptr %[[VAL_38]], ptr @.offload_sizes.1, ptr @.offload_maptypes.2, ptr @.offload_mapnames.3, ptr null) // CHECK: br label %[[VAL_39:.*]] -// CHECK: omp_if.else5: ; preds = %[[VAL_25]] +// CHECK: omp_if.else8: ; preds = %[[VAL_25]] // CHECK: br label %[[VAL_39]] -// CHECK: omp_if.end6: ; preds = %[[VAL_30]], %[[VAL_29]] +// CHECK: omp_if.end9: ; preds = %[[VAL_30]], %[[VAL_29]] // CHECK: ret void // ----- diff --git a/openmp/libomptarget/test/offloading/fortran/target-map-enter-exit-array-2.f90 b/openmp/libomptarget/test/offloading/fortran/target-map-enter-exit-array-2.f90 new file mode 100644 index 000000000000..489c2532a762 --- /dev/null +++ b/openmp/libomptarget/test/offloading/fortran/target-map-enter-exit-array-2.f90 @@ -0,0 +1,39 @@ +! Offloading test checking interaction of an +! enter and exit map of an array of scalars +! REQUIRES: flang, amdgcn-amd-amdhsa +! UNSUPPORTED: nvptx64-nvidia-cuda +! UNSUPPORTED: nvptx64-nvidia-cuda-LTO +! UNSUPPORTED: aarch64-unknown-linux-gnu +! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO +! UNSUPPORTED: x86_64-pc-linux-gnu +! UNSUPPORTED: x86_64-pc-linux-gnu-LTO + +! RUN: %libomptarget-compile-fortran-run-and-check-generic +program main + integer :: array(10) + + do I = 1, 10 + array(I) = I + I + end do + + !$omp target enter data map(to: array) + + ! Shouldn't overwrite data already locked in + ! on target via enter, this will then be + ! overwritten by our exit + do I = 1, 10 + array(I) = 10 + end do + + !$omp target + do i=1,10 + array(i) = array(i) + i + end do + !$omp end target + + !$omp target exit data map(from: array) + + print*, array +end program + +!CHECK: 3 6 9 12 15 18 21 24 27 30 diff --git a/openmp/libomptarget/test/offloading/fortran/target-map-enter-exit-array-bounds.f90 b/openmp/libomptarget/test/offloading/fortran/target-map-enter-exit-array-bounds.f90 new file mode 100644 index 000000000000..3c8c3507ed72 --- /dev/null +++ b/openmp/libomptarget/test/offloading/fortran/target-map-enter-exit-array-bounds.f90 @@ -0,0 +1,44 @@ +! Offloading test checking interaction of an +! enter and exit map of an array of scalars +! with specified bounds +! REQUIRES: flang, amdgcn-amd-amdhsa +! UNSUPPORTED: nvptx64-nvidia-cuda +! UNSUPPORTED: nvptx64-nvidia-cuda-LTO +! UNSUPPORTED: aarch64-unknown-linux-gnu +! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO +! UNSUPPORTED: x86_64-pc-linux-gnu +! UNSUPPORTED: x86_64-pc-linux-gnu-LTO + +! RUN: %libomptarget-compile-fortran-run-and-check-generic + +program main + integer :: array(10) + + do I = 1, 10 + array(I) = I + I + end do + + !$omp target enter data map(to: array(3:6)) + + ! Shouldn't overwrite data already locked in + ! on target via enter, which will then be + ! overwritten by our exit + do I = 1, 10 + array(I) = 10 + end do + + ! The compiler/runtime is less lenient about read/write out of + ! bounds when using enter and exit, we have to specifically loop + ! over the correctly mapped range + !$omp target + do i=3,6 + array(i) = array(i) + i + end do + !$omp end target + + !$omp target exit data map(from: array(3:6)) + + print *, array +end program + +!CHECK: 10 10 9 12 15 18 10 10 10 10 diff --git a/openmp/libomptarget/test/offloading/fortran/target-map-enter-exit-scalar.f90 b/openmp/libomptarget/test/offloading/fortran/target-map-enter-exit-scalar.f90 new file mode 100644 index 000000000000..29a0b5ee3e62 --- /dev/null +++ b/openmp/libomptarget/test/offloading/fortran/target-map-enter-exit-scalar.f90 @@ -0,0 +1,33 @@ +! Offloading test checking interaction of an +! enter and exit map of an scalar +! REQUIRES: flang, amdgcn-amd-amdhsa +! UNSUPPORTED: nvptx64-nvidia-cuda +! UNSUPPORTED: nvptx64-nvidia-cuda-LTO +! UNSUPPORTED: aarch64-unknown-linux-gnu +! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO +! UNSUPPORTED: x86_64-pc-linux-gnu +! UNSUPPORTED: x86_64-pc-linux-gnu-LTO + +! RUN: %libomptarget-compile-fortran-run-and-check-generic +program main + integer :: scalar + scalar = 10 + + !$omp target enter data map(to: scalar) + + !ignored, as we've already attached + scalar = 20 + + !$omp target + scalar = scalar + 50 + !$omp end target + + !$omp target exit data map(from: scalar) + + ! not the answer one may expect, but it is the same + ! answer Clang gives so we are correctly on par with + ! Clang for the moment. + print *, scalar +end program + +!CHECK: 10 -- GitLab From 71db97152173a524a3e16e02b7fdc50f405c8695 Mon Sep 17 00:00:00 2001 From: Matthias Gehre Date: Fri, 22 Mar 2024 15:39:52 +0100 Subject: [PATCH 060/306] [mlir][emitc] Arith to EmitC: Handle addi, subi and muli (#86120) Important to consider that `arith` has wrap around semantics, and in C++ signed overflow is UB. Unless the operation guarantees that no signed overflow happens, we will perform the arithmetic in an equivalent unsigned type. `bool` also doesn't wrap around in C++, and is not addressed here. --- .../Conversion/ArithToEmitC/ArithToEmitC.cpp | 52 +++++++++++++++++++ .../ArithToEmitC/arith-to-emitc-failed.mlir | 15 ++++++ .../ArithToEmitC/arith-to-emitc.mlir | 51 ++++++++++++++++++ 3 files changed, 118 insertions(+) create mode 100644 mlir/test/Conversion/ArithToEmitC/arith-to-emitc-failed.mlir diff --git a/mlir/lib/Conversion/ArithToEmitC/ArithToEmitC.cpp b/mlir/lib/Conversion/ArithToEmitC/ArithToEmitC.cpp index 3532785c31b9..db493c1294ba 100644 --- a/mlir/lib/Conversion/ArithToEmitC/ArithToEmitC.cpp +++ b/mlir/lib/Conversion/ArithToEmitC/ArithToEmitC.cpp @@ -55,6 +55,55 @@ public: } }; +template +class IntegerOpConversion final : public OpConversionPattern { +public: + using OpConversionPattern::OpConversionPattern; + + LogicalResult + matchAndRewrite(ArithOp op, typename ArithOp::Adaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + + Type type = this->getTypeConverter()->convertType(op.getType()); + if (!isa_and_nonnull(type)) { + return rewriter.notifyMatchFailure(op, "expected integer type"); + } + + if (type.isInteger(1)) { + // arith expects wrap-around arithmethic, which doesn't happen on `bool`. + return rewriter.notifyMatchFailure(op, "i1 type is not implemented"); + } + + Value lhs = adaptor.getLhs(); + Value rhs = adaptor.getRhs(); + Type arithmeticType = type; + if ((type.isSignlessInteger() || type.isSignedInteger()) && + !bitEnumContainsAll(op.getOverflowFlags(), + arith::IntegerOverflowFlags::nsw)) { + // If the C type is signed and the op doesn't guarantee "No Signed Wrap", + // we compute in unsigned integers to avoid UB. + arithmeticType = rewriter.getIntegerType(type.getIntOrFloatBitWidth(), + /*isSigned=*/false); + } + if (arithmeticType != type) { + lhs = rewriter.template create(op.getLoc(), arithmeticType, + lhs); + rhs = rewriter.template create(op.getLoc(), arithmeticType, + rhs); + } + + Value result = rewriter.template create(op.getLoc(), + arithmeticType, lhs, rhs); + + if (arithmeticType != type) { + result = + rewriter.template create(op.getLoc(), type, result); + } + rewriter.replaceOp(op, result); + return success(); + } +}; + class SelectOpConversion : public OpConversionPattern { public: using OpConversionPattern::OpConversionPattern; @@ -96,6 +145,9 @@ void mlir::populateArithToEmitCPatterns(TypeConverter &typeConverter, ArithOpConversion, ArithOpConversion, ArithOpConversion, + IntegerOpConversion, + IntegerOpConversion, + IntegerOpConversion, SelectOpConversion >(typeConverter, ctx); // clang-format on diff --git a/mlir/test/Conversion/ArithToEmitC/arith-to-emitc-failed.mlir b/mlir/test/Conversion/ArithToEmitC/arith-to-emitc-failed.mlir new file mode 100644 index 000000000000..30abd81f3d44 --- /dev/null +++ b/mlir/test/Conversion/ArithToEmitC/arith-to-emitc-failed.mlir @@ -0,0 +1,15 @@ +// RUN: mlir-opt -convert-arith-to-emitc %s -split-input-file -verify-diagnostics + +func.func @bool(%arg0: i1, %arg1: i1) { + // expected-error@+1 {{failed to legalize operation 'arith.addi'}} + %0 = arith.addi %arg0, %arg1 : i1 + return +} + +// ----- + +func.func @vector(%arg0: vector<4xi32>, %arg1: vector<4xi32>) { + // expected-error@+1 {{failed to legalize operation 'arith.addi'}} + %0 = arith.addi %arg0, %arg1 : vector<4xi32> + return +} diff --git a/mlir/test/Conversion/ArithToEmitC/arith-to-emitc.mlir b/mlir/test/Conversion/ArithToEmitC/arith-to-emitc.mlir index 022530ef4db8..76ba518577ab 100644 --- a/mlir/test/Conversion/ArithToEmitC/arith-to-emitc.mlir +++ b/mlir/test/Conversion/ArithToEmitC/arith-to-emitc.mlir @@ -37,6 +37,57 @@ func.func @arith_ops(%arg0: f32, %arg1: f32) { // ----- +// CHECK-LABEL: arith_integer_ops +func.func @arith_integer_ops(%arg0: i32, %arg1: i32) { + // CHECK: %[[C1:[^ ]*]] = emitc.cast %arg0 : i32 to ui32 + // CHECK: %[[C2:[^ ]*]] = emitc.cast %arg1 : i32 to ui32 + // CHECK: %[[ADD:[^ ]*]] = emitc.add %[[C1]], %[[C2]] : (ui32, ui32) -> ui32 + // CHECK: %[[C3:[^ ]*]] = emitc.cast %[[ADD]] : ui32 to i32 + %0 = arith.addi %arg0, %arg1 : i32 + // CHECK: %[[C1:[^ ]*]] = emitc.cast %arg0 : i32 to ui32 + // CHECK: %[[C2:[^ ]*]] = emitc.cast %arg1 : i32 to ui32 + // CHECK: %[[SUB:[^ ]*]] = emitc.sub %[[C1]], %[[C2]] : (ui32, ui32) -> ui32 + // CHECK: %[[C3:[^ ]*]] = emitc.cast %[[SUB]] : ui32 to i32 + %1 = arith.subi %arg0, %arg1 : i32 + // CHECK: %[[C1:[^ ]*]] = emitc.cast %arg0 : i32 to ui32 + // CHECK: %[[C2:[^ ]*]] = emitc.cast %arg1 : i32 to ui32 + // CHECK: %[[MUL:[^ ]*]] = emitc.mul %[[C1]], %[[C2]] : (ui32, ui32) -> ui32 + // CHECK: %[[C3:[^ ]*]] = emitc.cast %[[MUL]] : ui32 to i32 + %2 = arith.muli %arg0, %arg1 : i32 + + return +} + +// ----- + +// CHECK-LABEL: arith_integer_ops_signed_nsw +func.func @arith_integer_ops_signed_nsw(%arg0: i32, %arg1: i32) { + // CHECK: emitc.add %arg0, %arg1 : (i32, i32) -> i32 + %0 = arith.addi %arg0, %arg1 overflow : i32 + // CHECK: emitc.sub %arg0, %arg1 : (i32, i32) -> i32 + %1 = arith.subi %arg0, %arg1 overflow : i32 + // CHECK: emitc.mul %arg0, %arg1 : (i32, i32) -> i32 + %2 = arith.muli %arg0, %arg1 overflow : i32 + + return +} + +// ----- + +// CHECK-LABEL: arith_index +func.func @arith_index(%arg0: index, %arg1: index) { + // CHECK: emitc.add %arg0, %arg1 : (index, index) -> index + %0 = arith.addi %arg0, %arg1 : index + // CHECK: emitc.sub %arg0, %arg1 : (index, index) -> index + %1 = arith.subi %arg0, %arg1 : index + // CHECK: emitc.mul %arg0, %arg1 : (index, index) -> index + %2 = arith.muli %arg0, %arg1 : index + + return +} + +// ----- + func.func @arith_select(%arg0: i1, %arg1: tensor<8xi32>, %arg2: tensor<8xi32>) -> () { // CHECK: [[V0:[^ ]*]] = emitc.conditional %arg0, %arg1, %arg2 : tensor<8xi32> %0 = arith.select %arg0, %arg1, %arg2 : i1, tensor<8xi32> -- GitLab From 9c0a0659d40f613e873e416833d2293365b48e06 Mon Sep 17 00:00:00 2001 From: Alexey Bataev Date: Fri, 22 Mar 2024 07:31:35 -0700 Subject: [PATCH 061/306] [SLP]Fix a crash for non-profitable non-schedulable single buildvector node tree, if the threshold allows its vectorization. --- .../Transforms/Vectorize/SLPVectorizer.cpp | 2 + .../small-tree-not-schedulable-bv-node.ll | 263 ++++++++++++++++++ 2 files changed, 265 insertions(+) create mode 100644 llvm/test/Transforms/SLPVectorizer/RISCV/small-tree-not-schedulable-bv-node.ll diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 7295ae0ba90b..0f7afa2fc25c 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -9278,6 +9278,8 @@ bool BoUpSLP::isTreeTinyAndNotFullyVectorizable(bool ForReduction) const { bool IsAllowedSingleBVNode = VectorizableTree.size() > 1 || (VectorizableTree.size() == 1 && VectorizableTree.front()->getOpcode() && + VectorizableTree.front()->getOpcode() != Instruction::PHI && + VectorizableTree.front()->getOpcode() != Instruction::GetElementPtr && allSameBlock(VectorizableTree.front()->Scalars)); if (any_of(VectorizableTree, [&](const std::unique_ptr &TE) { return TE->State == TreeEntry::NeedToGather && diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/small-tree-not-schedulable-bv-node.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/small-tree-not-schedulable-bv-node.ll new file mode 100644 index 000000000000..26f3fcae3a33 --- /dev/null +++ b/llvm/test/Transforms/SLPVectorizer/RISCV/small-tree-not-schedulable-bv-node.ll @@ -0,0 +1,263 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -S -mtriple=riscv64-unknown-linux-gnu -slp-threshold=-100 -mattr=+v < %s | FileCheck %s + +define void @test1() personality ptr null { +; CHECK-LABEL: define void @test1( +; CHECK-SAME: ) #[[ATTR0:[0-9]+]] personality ptr null { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[CALL33:%.*]] = invoke ptr null(i64 0, ptr null) +; CHECK-NEXT: to label [[INVOKE_CONT32:%.*]] unwind label [[LPAD31_LOOPEXIT:%.*]] +; CHECK: invoke.cont32: +; CHECK-NEXT: invoke void null(ptr null, ptr null) +; CHECK-NEXT: to label [[INVOKE_CONT37:%.*]] unwind label [[LPAD34_LOOPEXIT:%.*]] +; CHECK: invoke.cont37: +; CHECK-NEXT: unreachable +; CHECK: lpad31.loopexit: +; CHECK-NEXT: [[LPAD_LOOPEXIT:%.*]] = landingpad { ptr, i32 } +; CHECK-NEXT: cleanup +; CHECK-NEXT: br label [[EHCLEANUP47:%.*]] +; CHECK: lpad34.loopexit: +; CHECK-NEXT: [[DOTLCSSA101:%.*]] = phi ptr [ null, [[INVOKE_CONT32]] ] +; CHECK-NEXT: [[CALL33_LCSSA96:%.*]] = phi ptr [ [[CALL33]], [[INVOKE_CONT32]] ] +; CHECK-NEXT: [[LPAD_LOOPEXIT56:%.*]] = landingpad { ptr, i32 } +; CHECK-NEXT: cleanup +; CHECK-NEXT: br label [[LPAD34_BODY:%.*]] +; CHECK: lpad34.loopexit.split-lp: +; CHECK-NEXT: [[LPAD_LOOPEXIT_SPLIT_LP57:%.*]] = landingpad { ptr, i32 } +; CHECK-NEXT: cleanup +; CHECK-NEXT: br label [[LPAD34_BODY]] +; CHECK: lpad34.body: +; CHECK-NEXT: [[TMP0:%.*]] = phi ptr [ [[DOTLCSSA101]], [[LPAD34_LOOPEXIT]] ], [ null, [[LPAD34_LOOPEXIT_SPLIT_LP:%.*]] ] +; CHECK-NEXT: [[CALL3399:%.*]] = phi ptr [ [[CALL33_LCSSA96]], [[LPAD34_LOOPEXIT]] ], [ null, [[LPAD34_LOOPEXIT_SPLIT_LP]] ] +; CHECK-NEXT: br label [[EHCLEANUP47]] +; CHECK: ehcleanup47: +; CHECK-NEXT: resume { ptr, i32 } zeroinitializer +; +entry: + %call33 = invoke ptr null(i64 0, ptr null) + to label %invoke.cont32 unwind label %lpad31.loopexit + +invoke.cont32: + invoke void null(ptr null, ptr null) + to label %invoke.cont37 unwind label %lpad34.loopexit + +invoke.cont37: + unreachable + +lpad31.loopexit: + %lpad.loopexit = landingpad { ptr, i32 } + cleanup + br label %ehcleanup47 + +lpad34.loopexit: + %.lcssa101 = phi ptr [ null, %invoke.cont32 ] + %call33.lcssa96 = phi ptr [ %call33, %invoke.cont32 ] + %lpad.loopexit56 = landingpad { ptr, i32 } + cleanup + br label %lpad34.body + +lpad34.loopexit.split-lp: + %lpad.loopexit.split-lp57 = landingpad { ptr, i32 } + cleanup + br label %lpad34.body + +lpad34.body: + %0 = phi ptr [ %.lcssa101, %lpad34.loopexit ], [ null, %lpad34.loopexit.split-lp ] + %call3399 = phi ptr [ %call33.lcssa96, %lpad34.loopexit ], [ null, %lpad34.loopexit.split-lp ] + br label %ehcleanup47 + +ehcleanup47: + resume { ptr, i32 } zeroinitializer +} + +define i32 @test2(i64 %idx.ext.i48.pre-phi) { +; CHECK-LABEL: define i32 @test2( +; CHECK-SAME: i64 [[IDX_EXT_I48_PRE_PHI:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: entry: +; CHECK-NEXT: br label [[DO_ACTION:%.*]] +; CHECK: do_action: +; CHECK-NEXT: switch i32 0, label [[SW_DEFAULT:%.*]] [ +; CHECK-NEXT: i32 1, label [[CLEANUP185:%.*]] +; CHECK-NEXT: i32 2, label [[CLEANUP185]] +; CHECK-NEXT: i32 0, label [[CLEANUP185]] +; CHECK-NEXT: i32 4, label [[CLEANUP185]] +; CHECK-NEXT: i32 5, label [[CLEANUP185]] +; CHECK-NEXT: i32 6, label [[CLEANUP185]] +; CHECK-NEXT: i32 7, label [[CLEANUP185]] +; CHECK-NEXT: i32 8, label [[CLEANUP185]] +; CHECK-NEXT: i32 9, label [[CLEANUP185]] +; CHECK-NEXT: i32 10, label [[CLEANUP185]] +; CHECK-NEXT: i32 11, label [[CLEANUP185]] +; CHECK-NEXT: i32 12, label [[CLEANUP185]] +; CHECK-NEXT: i32 13, label [[CLEANUP185]] +; CHECK-NEXT: i32 14, label [[CLEANUP185]] +; CHECK-NEXT: i32 15, label [[CLEANUP185]] +; CHECK-NEXT: i32 16, label [[CLEANUP185]] +; CHECK-NEXT: i32 17, label [[CLEANUP185]] +; CHECK-NEXT: i32 18, label [[CLEANUP185]] +; CHECK-NEXT: i32 19, label [[CLEANUP185]] +; CHECK-NEXT: i32 20, label [[CLEANUP185]] +; CHECK-NEXT: i32 21, label [[CLEANUP185]] +; CHECK-NEXT: i32 22, label [[CLEANUP185]] +; CHECK-NEXT: i32 23, label [[CLEANUP185]] +; CHECK-NEXT: i32 24, label [[CLEANUP185]] +; CHECK-NEXT: i32 25, label [[CLEANUP185]] +; CHECK-NEXT: i32 26, label [[CLEANUP185]] +; CHECK-NEXT: i32 27, label [[CLEANUP185]] +; CHECK-NEXT: i32 28, label [[CLEANUP185]] +; CHECK-NEXT: i32 29, label [[CLEANUP185]] +; CHECK-NEXT: i32 30, label [[CLEANUP185]] +; CHECK-NEXT: i32 31, label [[CLEANUP185]] +; CHECK-NEXT: i32 32, label [[CLEANUP185]] +; CHECK-NEXT: i32 33, label [[CLEANUP185]] +; CHECK-NEXT: i32 34, label [[CLEANUP185]] +; CHECK-NEXT: i32 35, label [[CLEANUP185]] +; CHECK-NEXT: i32 36, label [[CLEANUP185]] +; CHECK-NEXT: i32 37, label [[CLEANUP185]] +; CHECK-NEXT: i32 38, label [[CLEANUP185]] +; CHECK-NEXT: i32 39, label [[CLEANUP185]] +; CHECK-NEXT: i32 40, label [[CLEANUP185]] +; CHECK-NEXT: i32 41, label [[CLEANUP185]] +; CHECK-NEXT: i32 42, label [[CLEANUP185]] +; CHECK-NEXT: i32 43, label [[CLEANUP185]] +; CHECK-NEXT: i32 44, label [[CLEANUP185]] +; CHECK-NEXT: i32 45, label [[CLEANUP185]] +; CHECK-NEXT: i32 46, label [[CLEANUP185]] +; CHECK-NEXT: i32 47, label [[CLEANUP185]] +; CHECK-NEXT: i32 48, label [[CLEANUP185]] +; CHECK-NEXT: i32 49, label [[CLEANUP185]] +; CHECK-NEXT: i32 50, label [[CLEANUP185]] +; CHECK-NEXT: i32 51, label [[CLEANUP185]] +; CHECK-NEXT: i32 52, label [[CLEANUP185]] +; CHECK-NEXT: i32 53, label [[CLEANUP185]] +; CHECK-NEXT: i32 54, label [[CLEANUP185]] +; CHECK-NEXT: i32 55, label [[CLEANUP185]] +; CHECK-NEXT: i32 56, label [[CLEANUP185]] +; CHECK-NEXT: i32 57, label [[DO_ACTION]] +; CHECK-NEXT: i32 58, label [[CLEANUP185]] +; CHECK-NEXT: i32 59, label [[CLEANUP185]] +; CHECK-NEXT: i32 60, label [[DO_ACTION]] +; CHECK-NEXT: i32 61, label [[DO_ACTION]] +; CHECK-NEXT: i32 62, label [[CLEANUP185]] +; CHECK-NEXT: i32 70, label [[SW_BB175:%.*]] +; CHECK-NEXT: i32 64, label [[CLEANUP185]] +; CHECK-NEXT: i32 65, label [[DO_ACTION]] +; CHECK-NEXT: i32 66, label [[DO_ACTION]] +; CHECK-NEXT: i32 67, label [[CLEANUP185]] +; CHECK-NEXT: i32 72, label [[CLEANUP185]] +; CHECK-NEXT: i32 69, label [[DO_ACTION]] +; CHECK-NEXT: i32 71, label [[CLEANUP185]] +; CHECK-NEXT: ] +; CHECK: yy_get_previous_state.exit.loopexit: +; CHECK-NEXT: br label [[YY_FIND_ACTION_BACKEDGE:%.*]] +; CHECK: yy_find_action.backedge: +; CHECK-NEXT: [[YY_BP_1_BE:%.*]] = phi ptr [ [[ADD_PTR_I49:%.*]], [[SW_BB175]] ], [ null, [[YY_GET_PREVIOUS_STATE_EXIT_LOOPEXIT:%.*]] ] +; CHECK-NEXT: [[YY_CP_2_BE:%.*]] = phi ptr [ [[ARRAYIDX178:%.*]], [[SW_BB175]] ], [ null, [[YY_GET_PREVIOUS_STATE_EXIT_LOOPEXIT]] ] +; CHECK-NEXT: br label [[DO_ACTION]] +; CHECK: sw.bb175: +; CHECK-NEXT: [[ARRAYIDX178]] = getelementptr i8, ptr null, i64 0 +; CHECK-NEXT: [[ADD_PTR_I49]] = getelementptr i8, ptr null, i64 [[IDX_EXT_I48_PRE_PHI]] +; CHECK-NEXT: [[CMP5_I50:%.*]] = icmp ult ptr [[ADD_PTR_I49]], [[ARRAYIDX178]] +; CHECK-NEXT: br label [[YY_FIND_ACTION_BACKEDGE]] +; CHECK: sw.default: +; CHECK-NEXT: unreachable +; CHECK: cleanup185: +; CHECK-NEXT: ret i32 0 +; +entry: + br label %do_action + +do_action: + switch i32 0, label %sw.default [ + i32 1, label %cleanup185 + i32 2, label %cleanup185 + i32 0, label %cleanup185 + i32 4, label %cleanup185 + i32 5, label %cleanup185 + i32 6, label %cleanup185 + i32 7, label %cleanup185 + i32 8, label %cleanup185 + i32 9, label %cleanup185 + i32 10, label %cleanup185 + i32 11, label %cleanup185 + i32 12, label %cleanup185 + i32 13, label %cleanup185 + i32 14, label %cleanup185 + i32 15, label %cleanup185 + i32 16, label %cleanup185 + i32 17, label %cleanup185 + i32 18, label %cleanup185 + i32 19, label %cleanup185 + i32 20, label %cleanup185 + i32 21, label %cleanup185 + i32 22, label %cleanup185 + i32 23, label %cleanup185 + i32 24, label %cleanup185 + i32 25, label %cleanup185 + i32 26, label %cleanup185 + i32 27, label %cleanup185 + i32 28, label %cleanup185 + i32 29, label %cleanup185 + i32 30, label %cleanup185 + i32 31, label %cleanup185 + i32 32, label %cleanup185 + i32 33, label %cleanup185 + i32 34, label %cleanup185 + i32 35, label %cleanup185 + i32 36, label %cleanup185 + i32 37, label %cleanup185 + i32 38, label %cleanup185 + i32 39, label %cleanup185 + i32 40, label %cleanup185 + i32 41, label %cleanup185 + i32 42, label %cleanup185 + i32 43, label %cleanup185 + i32 44, label %cleanup185 + i32 45, label %cleanup185 + i32 46, label %cleanup185 + i32 47, label %cleanup185 + i32 48, label %cleanup185 + i32 49, label %cleanup185 + i32 50, label %cleanup185 + i32 51, label %cleanup185 + i32 52, label %cleanup185 + i32 53, label %cleanup185 + i32 54, label %cleanup185 + i32 55, label %cleanup185 + i32 56, label %cleanup185 + i32 57, label %do_action + i32 58, label %cleanup185 + i32 59, label %cleanup185 + i32 60, label %do_action + i32 61, label %do_action + i32 62, label %cleanup185 + i32 70, label %sw.bb175 + i32 64, label %cleanup185 + i32 65, label %do_action + i32 66, label %do_action + i32 67, label %cleanup185 + i32 72, label %cleanup185 + i32 69, label %do_action + i32 71, label %cleanup185 + ] + +yy_get_previous_state.exit.loopexit: + br label %yy_find_action.backedge + +yy_find_action.backedge: + %yy_bp.1.be = phi ptr [ %add.ptr.i49, %sw.bb175 ], [ null, %yy_get_previous_state.exit.loopexit ] + %yy_cp.2.be = phi ptr [ %arrayidx178, %sw.bb175 ], [ null, %yy_get_previous_state.exit.loopexit ] + br label %do_action + +sw.bb175: + %arrayidx178 = getelementptr i8, ptr null, i64 0 + %add.ptr.i49 = getelementptr i8, ptr null, i64 %idx.ext.i48.pre-phi + %cmp5.i50 = icmp ult ptr %add.ptr.i49, %arrayidx178 + br label %yy_find_action.backedge + +sw.default: + unreachable + +cleanup185: + ret i32 0 +} -- GitLab From 6f44bb7717897191be25aa01161831c67cdf5b84 Mon Sep 17 00:00:00 2001 From: Antonio Frighetto Date: Fri, 22 Mar 2024 15:40:15 +0100 Subject: [PATCH 062/306] [Object] Ensure header size not to underflow in `OffloadBinary::create` Prevent potential integer underflows when header size is not valid. Fixes: https://github.com/llvm/llvm-project/issues/86280. --- llvm/lib/Object/OffloadBinary.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/llvm/lib/Object/OffloadBinary.cpp b/llvm/lib/Object/OffloadBinary.cpp index 4ab6536dc90b..6e9f8bed513c 100644 --- a/llvm/lib/Object/OffloadBinary.cpp +++ b/llvm/lib/Object/OffloadBinary.cpp @@ -189,7 +189,10 @@ OffloadBinary::create(MemoryBufferRef Buf) { return errorCodeToError(object_error::parse_failed); if (TheHeader->Size > Buf.getBufferSize() || - TheHeader->EntryOffset > TheHeader->Size - sizeof(Entry) || + TheHeader->Size < sizeof(Entry) || TheHeader->Size < sizeof(Header)) + return errorCodeToError(object_error::unexpected_eof); + + if (TheHeader->EntryOffset > TheHeader->Size - sizeof(Entry) || TheHeader->EntrySize > TheHeader->Size - sizeof(Header)) return errorCodeToError(object_error::unexpected_eof); -- GitLab From b44771f480385fa93ba7719a57e759e19747e709 Mon Sep 17 00:00:00 2001 From: Wang Pengcheng Date: Fri, 22 Mar 2024 18:24:23 +0800 Subject: [PATCH 063/306] [RISCV] Support RISC-V Profiles in -march option (#76357) This PR implements the draft https://github.com/riscv-non-isa/riscv-toolchain-conventions/pull/36. Currently, we replace specified profile in `-march` with standard arch string. This is recommitted as 66f88de was reverted because of failures caused by lacking `--target` option. --- clang/docs/ReleaseNotes.rst | 1 + clang/test/Driver/riscv-profiles.c | 324 +++++++++++++++++++++++++++++ llvm/lib/Support/RISCVISAInfo.cpp | 64 ++++++ 3 files changed, 389 insertions(+) create mode 100644 clang/test/Driver/riscv-profiles.c diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index 45b2e01af997..d6e179ca9d69 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -520,6 +520,7 @@ RISC-V Support ^^^^^^^^^^^^^^ - ``__attribute__((rvv_vector_bits(N)))`` is now supported for RVV vbool*_t types. +- Profile names in ``-march`` option are now supported. CUDA/HIP Language Changes ^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/clang/test/Driver/riscv-profiles.c b/clang/test/Driver/riscv-profiles.c new file mode 100644 index 000000000000..0227487015ba --- /dev/null +++ b/clang/test/Driver/riscv-profiles.c @@ -0,0 +1,324 @@ +// RUN: %clang --target=riscv32 -### -c %s 2>&1 -march=rvi20u32 \ +// RUN: | FileCheck -check-prefix=RVI20U32 %s +// RVI20U32: "-target-feature" "-a" +// RVI20U32: "-target-feature" "-c" +// RVI20U32: "-target-feature" "-d" +// RVI20U32: "-target-feature" "-f" +// RVI20U32: "-target-feature" "-m" + +// RUN: %clang --target=riscv64 -### -c %s 2>&1 -march=rvi20u64 \ +// RUN: | FileCheck -check-prefix=RVI20U64 %s +// RVI20U64: "-target-feature" "-a" +// RVI20U64: "-target-feature" "-c" +// RVI20U64: "-target-feature" "-d" +// RVI20U64: "-target-feature" "-f" +// RVI20U64: "-target-feature" "-m" + +// RUN: %clang --target=riscv64 -### -c %s 2>&1 -march=rva20u64 \ +// RUN: | FileCheck -check-prefix=RVA20U64 %s +// RVA20U64: "-target-feature" "+m" +// RVA20U64: "-target-feature" "+a" +// RVA20U64: "-target-feature" "+f" +// RVA20U64: "-target-feature" "+d" +// RVA20U64: "-target-feature" "+c" +// RVA20U64: "-target-feature" "+ziccamoa" +// RVA20U64: "-target-feature" "+ziccif" +// RVA20U64: "-target-feature" "+zicclsm" +// RVA20U64: "-target-feature" "+ziccrse" +// RVA20U64: "-target-feature" "+zicntr" +// RVA20U64: "-target-feature" "+zicsr" +// RVA20U64: "-target-feature" "+za128rs" + +// RUN: %clang --target=riscv64 -### -c %s 2>&1 -march=rva20s64 \ +// RUN: | FileCheck -check-prefix=RVA20S64 %s +// RVA20S64: "-target-feature" "+m" +// RVA20S64: "-target-feature" "+a" +// RVA20S64: "-target-feature" "+f" +// RVA20S64: "-target-feature" "+d" +// RVA20S64: "-target-feature" "+c" +// RVA20S64: "-target-feature" "+ziccamoa" +// RVA20S64: "-target-feature" "+ziccif" +// RVA20S64: "-target-feature" "+zicclsm" +// RVA20S64: "-target-feature" "+ziccrse" +// RVA20S64: "-target-feature" "+zicntr" +// RVA20S64: "-target-feature" "+zicsr" +// RVA20S64: "-target-feature" "+zifencei" +// RVA20S64: "-target-feature" "+za128rs" +// RVA20S64: "-target-feature" "+ssccptr" +// RVA20S64: "-target-feature" "+sstvala" +// RVA20S64: "-target-feature" "+sstvecd" +// RVA20S64: "-target-feature" "+svade" +// RVA20S64: "-target-feature" "+svbare" + +// RUN: %clang --target=riscv64 --target=riscv64 -### -c %s 2>&1 -march=rva22u64 \ +// RUN: | FileCheck -check-prefix=RVA22U64 %s +// RVA22U64: "-target-feature" "+m" +// RVA22U64: "-target-feature" "+a" +// RVA22U64: "-target-feature" "+f" +// RVA22U64: "-target-feature" "+d" +// RVA22U64: "-target-feature" "+c" +// RVA22U64: "-target-feature" "+zic64b" +// RVA22U64: "-target-feature" "+zicbom" +// RVA22U64: "-target-feature" "+zicbop" +// RVA22U64: "-target-feature" "+zicboz" +// RVA22U64: "-target-feature" "+ziccamoa" +// RVA22U64: "-target-feature" "+ziccif" +// RVA22U64: "-target-feature" "+zicclsm" +// RVA22U64: "-target-feature" "+ziccrse" +// RVA22U64: "-target-feature" "+zicntr" +// RVA22U64: "-target-feature" "+zicsr" +// RVA22U64: "-target-feature" "+zihintpause" +// RVA22U64: "-target-feature" "+zihpm" +// RVA22U64: "-target-feature" "+za64rs" +// RVA22U64: "-target-feature" "+zfhmin" +// RVA22U64: "-target-feature" "+zba" +// RVA22U64: "-target-feature" "+zbb" +// RVA22U64: "-target-feature" "+zbs" +// RVA22U64: "-target-feature" "+zkt" + +// RUN: %clang --target=riscv64 --target=riscv64 -### -c %s 2>&1 -march=rva22s64 \ +// RUN: | FileCheck -check-prefix=RVA22S64 %s +// RVA22S64: "-target-feature" "+m" +// RVA22S64: "-target-feature" "+a" +// RVA22S64: "-target-feature" "+f" +// RVA22S64: "-target-feature" "+d" +// RVA22S64: "-target-feature" "+c" +// RVA22S64: "-target-feature" "+zic64b" +// RVA22S64: "-target-feature" "+zicbom" +// RVA22S64: "-target-feature" "+zicbop" +// RVA22S64: "-target-feature" "+zicboz" +// RVA22S64: "-target-feature" "+ziccamoa" +// RVA22S64: "-target-feature" "+ziccif" +// RVA22S64: "-target-feature" "+zicclsm" +// RVA22S64: "-target-feature" "+ziccrse" +// RVA22S64: "-target-feature" "+zicntr" +// RVA22S64: "-target-feature" "+zicsr" +// RVA22S64: "-target-feature" "+zifencei" +// RVA22S64: "-target-feature" "+zihintpause" +// RVA22S64: "-target-feature" "+zihpm" +// RVA22S64: "-target-feature" "+za64rs" +// RVA22S64: "-target-feature" "+zfhmin" +// RVA22S64: "-target-feature" "+zba" +// RVA22S64: "-target-feature" "+zbb" +// RVA22S64: "-target-feature" "+zbs" +// RVA22S64: "-target-feature" "+zkt" +// RVA22S64: "-target-feature" "+ssccptr" +// RVA22S64: "-target-feature" "+sscounterenw" +// RVA22S64: "-target-feature" "+sstvala" +// RVA22S64: "-target-feature" "+sstvecd" +// RVA22S64: "-target-feature" "+svade" +// RVA22S64: "-target-feature" "+svbare" +// RVA22S64: "-target-feature" "+svinval" +// RVA22S64: "-target-feature" "+svpbmt" + +// RUN: %clang --target=riscv64 --target=riscv64 -### -c %s 2>&1 -march=rva23u64 -menable-experimental-extensions \ +// RUN: | FileCheck -check-prefix=RVA23U64 %s +// RVA23U64: "-target-feature" "+m" +// RVA23U64: "-target-feature" "+a" +// RVA23U64: "-target-feature" "+f" +// RVA23U64: "-target-feature" "+d" +// RVA23U64: "-target-feature" "+c" +// RVA23U64: "-target-feature" "+v" +// RVA23U64: "-target-feature" "+zic64b" +// RVA23U64: "-target-feature" "+zicbom" +// RVA23U64: "-target-feature" "+zicbop" +// RVA23U64: "-target-feature" "+zicboz" +// RVA23U64: "-target-feature" "+ziccamoa" +// RVA23U64: "-target-feature" "+ziccif" +// RVA23U64: "-target-feature" "+zicclsm" +// RVA23U64: "-target-feature" "+ziccrse" +// RVA23U64: "-target-feature" "+zicntr" +// RVA23U64: "-target-feature" "+zicond" +// RVA23U64: "-target-feature" "+zicsr" +// RVA23U64: "-target-feature" "+zihintntl" +// RVA23U64: "-target-feature" "+zihintpause" +// RVA23U64: "-target-feature" "+zihpm" +// RVA23U64: "-target-feature" "+experimental-zimop" +// RVA23U64: "-target-feature" "+za64rs" +// RVA23U64: "-target-feature" "+zawrs" +// RVA23U64: "-target-feature" "+zfa" +// RVA23U64: "-target-feature" "+zfhmin" +// RVA23U64: "-target-feature" "+zcb" +// RVA23U64: "-target-feature" "+experimental-zcmop" +// RVA23U64: "-target-feature" "+zba" +// RVA23U64: "-target-feature" "+zbb" +// RVA23U64: "-target-feature" "+zbs" +// RVA23U64: "-target-feature" "+zkt" +// RVA23U64: "-target-feature" "+zvbb" +// RVA23U64: "-target-feature" "+zvfhmin" +// RVA23U64: "-target-feature" "+zvkt" + +// RUN: %clang --target=riscv64 -### -c %s 2>&1 -march=rva23s64 -menable-experimental-extensions \ +// RUN: | FileCheck -check-prefix=RVA23S64 %s +// RVA23S64: "-target-feature" "+m" +// RVA23S64: "-target-feature" "+a" +// RVA23S64: "-target-feature" "+f" +// RVA23S64: "-target-feature" "+d" +// RVA23S64: "-target-feature" "+c" +// RVA23S64: "-target-feature" "+v" +// RVA23S64: "-target-feature" "+h" +// RVA23S64: "-target-feature" "+zic64b" +// RVA23S64: "-target-feature" "+zicbom" +// RVA23S64: "-target-feature" "+zicbop" +// RVA23S64: "-target-feature" "+zicboz" +// RVA23S64: "-target-feature" "+ziccamoa" +// RVA23S64: "-target-feature" "+ziccif" +// RVA23S64: "-target-feature" "+zicclsm" +// RVA23S64: "-target-feature" "+ziccrse" +// RVA23S64: "-target-feature" "+zicntr" +// RVA23S64: "-target-feature" "+zicond" +// RVA23S64: "-target-feature" "+zicsr" +// RVA23S64: "-target-feature" "+zifencei" +// RVA23S64: "-target-feature" "+zihintntl" +// RVA23S64: "-target-feature" "+zihintpause" +// RVA23S64: "-target-feature" "+zihpm" +// RVA23S64: "-target-feature" "+experimental-zimop" +// RVA23S64: "-target-feature" "+za64rs" +// RVA23S64: "-target-feature" "+zawrs" +// RVA23S64: "-target-feature" "+zfa" +// RVA23S64: "-target-feature" "+zfhmin" +// RVA23S64: "-target-feature" "+zcb" +// RVA23S64: "-target-feature" "+experimental-zcmop" +// RVA23S64: "-target-feature" "+zba" +// RVA23S64: "-target-feature" "+zbb" +// RVA23S64: "-target-feature" "+zbs" +// RVA23S64: "-target-feature" "+zkt" +// RVA23S64: "-target-feature" "+zvbb" +// RVA23S64: "-target-feature" "+zvfhmin" +// RVA23S64: "-target-feature" "+zvkt" +// RVA23S64: "-target-feature" "+shcounterenw" +// RVA23S64: "-target-feature" "+shgatpa" +// RVA23S64: "-target-feature" "+shtvala" +// RVA23S64: "-target-feature" "+shvsatpa" +// RVA23S64: "-target-feature" "+shvstvala" +// RVA23S64: "-target-feature" "+shvstvecd" +// RVA23S64: "-target-feature" "+ssccptr" +// RVA23S64: "-target-feature" "+sscofpmf" +// RVA23S64: "-target-feature" "+sscounterenw" +// RVA23S64: "-target-feature" "+experimental-ssnpm" +// RVA23S64: "-target-feature" "+ssstateen" +// RVA23S64: "-target-feature" "+sstc" +// RVA23S64: "-target-feature" "+sstvala" +// RVA23S64: "-target-feature" "+sstvecd" +// RVA23S64: "-target-feature" "+ssu64xl" +// RVA23S64: "-target-feature" "+svade" +// RVA23S64: "-target-feature" "+svbare" +// RVA23S64: "-target-feature" "+svinval" +// RVA23S64: "-target-feature" "+svnapot" +// RVA23S64: "-target-feature" "+svpbmt" + +// RUN: %clang --target=riscv64 -### -c %s 2>&1 -march=rvb23u64 -menable-experimental-extensions \ +// RUN: | FileCheck -check-prefix=RVB23U64 %s +// RVB23U64: "-target-feature" "+m" +// RVB23U64: "-target-feature" "+a" +// RVB23U64: "-target-feature" "+f" +// RVB23U64: "-target-feature" "+d" +// RVB23U64: "-target-feature" "+c" +// RVB23U64: "-target-feature" "+zic64b" +// RVB23U64: "-target-feature" "+zicbom" +// RVB23U64: "-target-feature" "+zicbop" +// RVB23U64: "-target-feature" "+zicboz" +// RVB23U64: "-target-feature" "+ziccamoa" +// RVB23U64: "-target-feature" "+ziccif" +// RVB23U64: "-target-feature" "+zicclsm" +// RVB23U64: "-target-feature" "+ziccrse" +// RVB23U64: "-target-feature" "+zicntr" +// RVB23U64: "-target-feature" "+zicond" +// RVB23U64: "-target-feature" "+zicsr" +// RVB23U64: "-target-feature" "+zihintntl" +// RVB23U64: "-target-feature" "+zihintpause" +// RVB23U64: "-target-feature" "+zihpm" +// RVB23U64: "-target-feature" "+experimental-zimop" +// RVB23U64: "-target-feature" "+za64rs" +// RVB23U64: "-target-feature" "+zawrs" +// RVB23U64: "-target-feature" "+zfa" +// RVB23U64: "-target-feature" "+zcb" +// RVB23U64: "-target-feature" "+experimental-zcmop" +// RVB23U64: "-target-feature" "+zba" +// RVB23U64: "-target-feature" "+zbb" +// RVB23U64: "-target-feature" "+zbs" +// RVB23U64: "-target-feature" "+zkt" + +// RUN: %clang --target=riscv64 -### -c %s 2>&1 -march=rvb23s64 -menable-experimental-extensions \ +// RUN: | FileCheck -check-prefix=RVB23S64 %s +// RVB23S64: "-target-feature" "+m" +// RVB23S64: "-target-feature" "+a" +// RVB23S64: "-target-feature" "+f" +// RVB23S64: "-target-feature" "+d" +// RVB23S64: "-target-feature" "+c" +// RVB23S64: "-target-feature" "+zic64b" +// RVB23S64: "-target-feature" "+zicbom" +// RVB23S64: "-target-feature" "+zicbop" +// RVB23S64: "-target-feature" "+zicboz" +// RVB23S64: "-target-feature" "+ziccamoa" +// RVB23S64: "-target-feature" "+ziccif" +// RVB23S64: "-target-feature" "+zicclsm" +// RVB23S64: "-target-feature" "+ziccrse" +// RVB23S64: "-target-feature" "+zicntr" +// RVB23S64: "-target-feature" "+zicond" +// RVB23S64: "-target-feature" "+zicsr" +// RVB23S64: "-target-feature" "+zifencei" +// RVB23S64: "-target-feature" "+zihintntl" +// RVB23S64: "-target-feature" "+zihintpause" +// RVB23S64: "-target-feature" "+zihpm" +// RVB23S64: "-target-feature" "+experimental-zimop" +// RVB23S64: "-target-feature" "+za64rs" +// RVB23S64: "-target-feature" "+zawrs" +// RVB23S64: "-target-feature" "+zfa" +// RVB23S64: "-target-feature" "+zcb" +// RVB23S64: "-target-feature" "+experimental-zcmop" +// RVB23S64: "-target-feature" "+zba" +// RVB23S64: "-target-feature" "+zbb" +// RVB23S64: "-target-feature" "+zbs" +// RVB23S64: "-target-feature" "+zkt" +// RVB23S64: "-target-feature" "+ssccptr" +// RVB23S64: "-target-feature" "+sscofpmf" +// RVB23S64: "-target-feature" "+sscounterenw" +// RVB23S64: "-target-feature" "+sstc" +// RVB23S64: "-target-feature" "+sstvala" +// RVB23S64: "-target-feature" "+sstvecd" +// RVB23S64: "-target-feature" "+ssu64xl" +// RVB23S64: "-target-feature" "+svade" +// RVB23S64: "-target-feature" "+svbare" +// RVB23S64: "-target-feature" "+svinval" +// RVB23S64: "-target-feature" "+svnapot" +// RVB23S64: "-target-feature" "+svpbmt" + +// RUN: %clang --target=riscv32 -### -c %s 2>&1 -march=rvm23u32 -menable-experimental-extensions \ +// RUN: | FileCheck -check-prefix=RVM23U32 %s +// RVM23U32: "-target-feature" "+m" +// RVM23U32: "-target-feature" "+zicbop" +// RVM23U32: "-target-feature" "+zicond" +// RVM23U32: "-target-feature" "+zicsr" +// RVM23U32: "-target-feature" "+zihintntl" +// RVM23U32: "-target-feature" "+zihintpause" +// RVM23U32: "-target-feature" "+experimental-zimop" +// RVM23U32: "-target-feature" "+zce" +// RVM23U32: "-target-feature" "+experimental-zcmop" +// RVM23U32: "-target-feature" "+zba" +// RVM23U32: "-target-feature" "+zbb" +// RVM23U32: "-target-feature" "+zbs" + +// RUN: %clang --target=riscv64 -### -c %s 2>&1 -march=rva22u64_zfa \ +// RUN: | FileCheck -check-prefix=PROFILE-WITH-ADDITIONAL %s +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+m" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+a" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+f" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+d" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+c" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zicbom" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zicbop" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zicboz" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zihintpause" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zfa" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zfhmin" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zba" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zbb" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zbs" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zkt" + +// RUN: not %clang --target=riscv64 -### -c %s 2>&1 -march=rva19u64_zfa | FileCheck -check-prefix=INVALID-PROFILE %s +// INVALID-PROFILE: error: invalid arch name 'rva19u64_zfa', unsupported profile + +// RUN: not %clang --target=riscv64 -### -c %s 2>&1 -march=rva22u64zfa | FileCheck -check-prefix=INVALID-ADDITIONAL %s +// INVALID-ADDITIONAL: error: invalid arch name 'rva22u64zfa', additional extensions must be after separator '_' diff --git a/llvm/lib/Support/RISCVISAInfo.cpp b/llvm/lib/Support/RISCVISAInfo.cpp index 39235ace4724..67e6e5b962b1 100644 --- a/llvm/lib/Support/RISCVISAInfo.cpp +++ b/llvm/lib/Support/RISCVISAInfo.cpp @@ -36,6 +36,11 @@ struct RISCVSupportedExtension { } }; +struct RISCVProfile { + StringLiteral Name; + StringLiteral MArch; +}; + } // end anonymous namespace static constexpr StringLiteral AllStdExts = "mafdqlcbkjtpvnh"; @@ -244,6 +249,42 @@ static const RISCVSupportedExtension SupportedExperimentalExtensions[] = { }; // clang-format on +static constexpr RISCVProfile SupportedProfiles[] = { + {"rvi20u32", "rv32i"}, + {"rvi20u64", "rv64i"}, + {"rva20u64", "rv64imafdc_ziccamoa_ziccif_zicclsm_ziccrse_zicntr_za128rs"}, + {"rva20s64", "rv64imafdc_ziccamoa_ziccif_zicclsm_ziccrse_zicntr_zifencei_" + "za128rs_ssccptr_sstvala_sstvecd_svade_svbare"}, + {"rva22u64", + "rv64imafdc_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_zicclsm_ziccrse_" + "zicntr_zihintpause_zihpm_za64rs_zfhmin_zba_zbb_zbs_zkt"}, + {"rva22s64", + "rv64imafdc_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_zicclsm_ziccrse_" + "zicntr_zifencei_zihintpause_zihpm_za64rs_zfhmin_zba_zbb_zbs_zkt_ssccptr_" + "sscounterenw_sstvala_sstvecd_svade_svbare_svinval_svpbmt"}, + {"rva23u64", + "rv64imafdcv_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_zicclsm_ziccrse_" + "zicntr_zicond_zihintntl_zihintpause_zihpm_zimop0p1_za64rs_zawrs_zfa_" + "zfhmin_zcb_zcmop0p2_zba_zbb_zbs_zkt_zvbb_zvfhmin_zvkt"}, + {"rva23s64", + "rv64imafdcvh_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_zicclsm_ziccrse_" + "zicntr_zicond_zifencei_zihintntl_zihintpause_zihpm_zimop0p1_za64rs_zawrs_" + "zfa_zfhmin_zcb_zcmop0p2_zba_zbb_zbs_zkt_zvbb_zvfhmin_zvkt_shcounterenw_" + "shgatpa_shtvala_shvsatpa_shvstvala_shvstvecd_ssccptr_sscofpmf_" + "sscounterenw_ssnpm0p8_ssstateen_sstc_sstvala_sstvecd_ssu64xl_svade_" + "svbare_svinval_svnapot_svpbmt"}, + {"rvb23u64", "rv64imafdc_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_" + "zicclsm_ziccrse_zicntr_zicond_zihintntl_zihintpause_zihpm_" + "zimop0p1_za64rs_zawrs_zfa_zcb_zcmop0p2_zba_zbb_zbs_zkt"}, + {"rvb23s64", + "rv64imafdc_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_zicclsm_ziccrse_" + "zicntr_zicond_zifencei_zihintntl_zihintpause_zihpm_zimop0p1_za64rs_zawrs_" + "zfa_zcb_zcmop0p2_zba_zbb_zbs_zkt_ssccptr_sscofpmf_sscounterenw_sstc_" + "sstvala_sstvecd_ssu64xl_svade_svbare_svinval_svnapot_svpbmt"}, + {"rvm23u32", "rv32im_zicbop_zicond_zicsr_zihintntl_zihintpause_zimop0p1_" + "zca_zcb_zce_zcmop0p2_zcmp_zcmt_zba_zbb_zbs"}, +}; + static void verifyTables() { #ifndef NDEBUG static std::atomic TableChecked(false); @@ -857,6 +898,29 @@ RISCVISAInfo::parseArchString(StringRef Arch, bool EnableExperimentalExtension, "string must be lowercase"); } + if (Arch.starts_with("rvi") || Arch.starts_with("rva") || + Arch.starts_with("rvb") || Arch.starts_with("rvm")) { + const auto *FoundProfile = + llvm::find_if(SupportedProfiles, [Arch](const RISCVProfile &Profile) { + return Arch.starts_with(Profile.Name); + }); + + if (FoundProfile == std::end(SupportedProfiles)) + return createStringError(errc::invalid_argument, "unsupported profile"); + + std::string NewArch = FoundProfile->MArch.str(); + StringRef ArchWithoutProfile = Arch.substr(FoundProfile->Name.size()); + if (!ArchWithoutProfile.empty()) { + if (!ArchWithoutProfile.starts_with("_")) + return createStringError( + errc::invalid_argument, + "additional extensions must be after separator '_'"); + NewArch += ArchWithoutProfile.str(); + } + return parseArchString(NewArch, EnableExperimentalExtension, + ExperimentalExtensionVersionCheck, IgnoreUnknown); + } + bool HasRV64 = Arch.starts_with("rv64"); // ISA string must begin with rv32 or rv64. if (!(Arch.starts_with("rv32") || HasRV64) || (Arch.size() < 5)) { -- GitLab From cdbec7baf1bc31b59526442c9d4d5f53aac746eb Mon Sep 17 00:00:00 2001 From: Nick Desaulniers Date: Fri, 22 Mar 2024 08:24:08 -0700 Subject: [PATCH 064/306] [libc] fix up the use of angle includes in include/ (#86027) Performed en-masse via: $ grep -rn "#include /#include "ll$1"/' $ grep -rn "#include <__" libc/include -l | \ xargs perl -pi -e 's/#include <__(.*)>/#include "__$1"/' Link: #83463 Link: #83210 --- libc/include/arpa/inet.h.def | 2 +- libc/include/assert.h.def | 2 +- libc/include/ctype.h.def | 2 +- libc/include/dirent.h.def | 2 +- libc/include/errno.h.def | 4 ++-- libc/include/fcntl.h.def | 4 ++-- libc/include/features.h.def | 4 ++-- libc/include/fenv.h.def | 4 ++-- libc/include/float.h.def | 2 +- libc/include/gpu/rpc.h.def | 4 ++-- libc/include/inttypes.h.def | 4 ++-- libc/include/limits.h.def | 2 +- .../llvm-libc-macros/containerof-macro.h | 2 +- .../llvm-libc-macros/sys-queue-macros.h | 4 ++-- libc/include/llvm-libc-types/__mutex_type.h | 2 +- .../llvm-libc-types/cookie_io_functions_t.h | 6 +++--- libc/include/llvm-libc-types/fd_set.h | 2 +- libc/include/llvm-libc-types/mtx_t.h | 2 +- libc/include/llvm-libc-types/once_flag.h | 2 +- libc/include/llvm-libc-types/pthread_attr_t.h | 2 +- .../include/llvm-libc-types/pthread_mutex_t.h | 2 +- libc/include/llvm-libc-types/pthread_once_t.h | 2 +- libc/include/llvm-libc-types/pthread_t.h | 2 +- libc/include/llvm-libc-types/siginfo_t.h | 8 ++++---- libc/include/llvm-libc-types/sigset_t.h | 2 +- libc/include/llvm-libc-types/stack_t.h | 2 +- libc/include/llvm-libc-types/struct_dirent.h | 4 ++-- .../llvm-libc-types/struct_epoll_event.h | 2 +- libc/include/llvm-libc-types/struct_rlimit.h | 2 +- libc/include/llvm-libc-types/struct_rusage.h | 2 +- .../llvm-libc-types/struct_sched_param.h | 6 +++--- .../llvm-libc-types/struct_sigaction.h | 4 ++-- .../include/llvm-libc-types/struct_sockaddr.h | 2 +- .../llvm-libc-types/struct_sockaddr_un.h | 2 +- libc/include/llvm-libc-types/struct_stat.h | 20 +++++++++---------- libc/include/llvm-libc-types/struct_termios.h | 6 +++--- .../include/llvm-libc-types/struct_timespec.h | 2 +- libc/include/llvm-libc-types/struct_timeval.h | 4 ++-- libc/include/llvm-libc-types/thrd_t.h | 2 +- libc/include/math.h.def | 6 +++--- libc/include/pthread.h.def | 2 +- libc/include/sched.h.def | 4 ++-- libc/include/search.h.def | 2 +- libc/include/setjmp.h.def | 2 +- libc/include/signal.h.def | 4 ++-- libc/include/spawn.h.def | 2 +- libc/include/stdbit.h.def | 4 ++-- libc/include/stdckdint.h.def | 4 ++-- libc/include/stdfix.h.def | 4 ++-- libc/include/stdint.h.def | 2 +- libc/include/stdio.h.def | 6 +++--- libc/include/stdlib.h.def | 4 ++-- libc/include/string.h.def | 4 ++-- libc/include/strings.h.def | 2 +- libc/include/sys/auxv.h.def | 4 ++-- libc/include/sys/epoll.h.def | 2 +- libc/include/sys/ioctl.h.def | 4 ++-- libc/include/sys/mman.h.def | 4 ++-- libc/include/sys/prctl.h.def | 2 +- libc/include/sys/queue.h | 2 +- libc/include/sys/random.h.def | 4 ++-- libc/include/sys/resource.h.def | 4 ++-- libc/include/sys/select.h.def | 4 ++-- libc/include/sys/sendfile.h.def | 2 +- libc/include/sys/socket.h.def | 4 ++-- libc/include/sys/stat.h.def | 4 ++-- libc/include/sys/time.h.def | 6 +++--- libc/include/sys/types.h.def | 2 +- libc/include/sys/utsname.h.def | 2 +- libc/include/sys/wait.h.def | 4 ++-- libc/include/termios.h.def | 4 ++-- libc/include/threads.h.def | 2 +- libc/include/time.h.def | 4 ++-- libc/include/uchar.h.def | 2 +- libc/include/unistd.h.def | 6 +++--- libc/include/wchar.h.def | 4 ++-- 76 files changed, 131 insertions(+), 131 deletions(-) diff --git a/libc/include/arpa/inet.h.def b/libc/include/arpa/inet.h.def index fdd5ae3e3f85..6a62b2c7be81 100644 --- a/libc/include/arpa/inet.h.def +++ b/libc/include/arpa/inet.h.def @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_ARPA_INET_H #define LLVM_LIBC_ARPA_INET_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" #include diff --git a/libc/include/assert.h.def b/libc/include/assert.h.def index e5d7dfbffdbb..e006133a7654 100644 --- a/libc/include/assert.h.def +++ b/libc/include/assert.h.def @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" // This file may be usefully included multiple times to change assert()'s // definition based on NDEBUG. diff --git a/libc/include/ctype.h.def b/libc/include/ctype.h.def index ac52a36bf72f..a9bb786931ea 100644 --- a/libc/include/ctype.h.def +++ b/libc/include/ctype.h.def @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_CTYPE_H #define LLVM_LIBC_CTYPE_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" %%public_api() diff --git a/libc/include/dirent.h.def b/libc/include/dirent.h.def index 3de8b1c6713f..6786578fbd06 100644 --- a/libc/include/dirent.h.def +++ b/libc/include/dirent.h.def @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_DIRENT_H #define LLVM_LIBC_DIRENT_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" %%public_api() diff --git a/libc/include/errno.h.def b/libc/include/errno.h.def index 90bd8bfecf2f..d7ae90ad4524 100644 --- a/libc/include/errno.h.def +++ b/libc/include/errno.h.def @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_ERRNO_H #define LLVM_LIBC_ERRNO_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" #ifdef __linux__ @@ -40,7 +40,7 @@ #endif // ENOTRECOVERABLE #else // __linux__ -#include +#include "llvm-libc-macros/generic-error-number-macros.h" #endif #if !defined(__AMDGPU__) && !defined(__NVPTX__) diff --git a/libc/include/fcntl.h.def b/libc/include/fcntl.h.def index b11645d18c5b..4f608845ce1e 100644 --- a/libc/include/fcntl.h.def +++ b/libc/include/fcntl.h.def @@ -9,8 +9,8 @@ #ifndef LLVM_LIBC_FCNTL_H #define LLVM_LIBC_FCNTL_H -#include <__llvm-libc-common.h> -#include +#include "__llvm-libc-common.h" +#include "llvm-libc-macros/fcntl-macros.h" %%public_api() diff --git a/libc/include/features.h.def b/libc/include/features.h.def index 64205f57acb5..238b88d4b90f 100644 --- a/libc/include/features.h.def +++ b/libc/include/features.h.def @@ -9,8 +9,8 @@ #ifndef LLVM_LIBC_FEATURES_H #define LLVM_LIBC_FEATURES_H -#include <__llvm-libc-common.h> -#include +#include "__llvm-libc-common.h" +#include "llvm-libc-macros/features-macros.h" %%public_api() diff --git a/libc/include/fenv.h.def b/libc/include/fenv.h.def index f131a44914ab..c677b2a5930d 100644 --- a/libc/include/fenv.h.def +++ b/libc/include/fenv.h.def @@ -9,8 +9,8 @@ #ifndef LLVM_LIBC_FENV_H #define LLVM_LIBC_FENV_H -#include <__llvm-libc-common.h> -#include +#include "__llvm-libc-common.h" +#include "llvm-libc-macros/fenv-macros.h" %%public_api() diff --git a/libc/include/float.h.def b/libc/include/float.h.def index 6d3599d78c69..3bcd7f5e3f98 100644 --- a/libc/include/float.h.def +++ b/libc/include/float.h.def @@ -9,6 +9,6 @@ #ifndef LLVM_LIBC_FLOAT_H #define LLVM_LIBC_FLOAT_H -#include +#include "llvm-libc-macros/float-macros.h" #endif // LLVM_LIBC_FLOAT_H diff --git a/libc/include/gpu/rpc.h.def b/libc/include/gpu/rpc.h.def index 0438cd65e7be..72acf0c81422 100644 --- a/libc/include/gpu/rpc.h.def +++ b/libc/include/gpu/rpc.h.def @@ -9,9 +9,9 @@ #ifndef LLVM_LIBC_GPU_RPC_H #define LLVM_LIBC_GPU_RPC_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" -#include +#include "llvm-libc-types/rpc_opcodes_t.h" %%public_api() diff --git a/libc/include/inttypes.h.def b/libc/include/inttypes.h.def index a99d4e931f51..5879d2d8e041 100644 --- a/libc/include/inttypes.h.def +++ b/libc/include/inttypes.h.def @@ -9,8 +9,8 @@ #ifndef LLVM_LIBC_INTTYPES_H #define LLVM_LIBC_INTTYPES_H -#include <__llvm-libc-common.h> -#include +#include "__llvm-libc-common.h" +#include "llvm-libc-macros/inttypes-macros.h" #include %%public_api() diff --git a/libc/include/limits.h.def b/libc/include/limits.h.def index de5f3490459e..c37c97c69a84 100644 --- a/libc/include/limits.h.def +++ b/libc/include/limits.h.def @@ -9,6 +9,6 @@ #ifndef LLVM_LIBC_LIMITS_H #define LLVM_LIBC_LIMITS_H -#include +#include "llvm-libc-macros/limits-macros.h" #endif // LLVM_LIBC_LIMITS_H diff --git a/libc/include/llvm-libc-macros/containerof-macro.h b/libc/include/llvm-libc-macros/containerof-macro.h index 62724abd3b0f..592acd6e3aa9 100644 --- a/libc/include/llvm-libc-macros/containerof-macro.h +++ b/libc/include/llvm-libc-macros/containerof-macro.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_MACROS_CONTAINEROF_MACRO_H #define LLVM_LIBC_MACROS_CONTAINEROF_MACRO_H -#include +#include "llvm-libc-macros/offsetof-macro.h" #define __containerof(ptr, type, member) \ ({ \ diff --git a/libc/include/llvm-libc-macros/sys-queue-macros.h b/libc/include/llvm-libc-macros/sys-queue-macros.h index fcac265333fc..089b6abaa024 100644 --- a/libc/include/llvm-libc-macros/sys-queue-macros.h +++ b/libc/include/llvm-libc-macros/sys-queue-macros.h @@ -9,8 +9,8 @@ #ifndef LLVM_LIBC_MACROS_SYS_QUEUE_MACROS_H #define LLVM_LIBC_MACROS_SYS_QUEUE_MACROS_H -#include -#include +#include "llvm-libc-macros/containerof-macro.h" +#include "llvm-libc-macros/null-macro.h" #ifdef __cplusplus #define QUEUE_TYPEOF(type) type diff --git a/libc/include/llvm-libc-types/__mutex_type.h b/libc/include/llvm-libc-types/__mutex_type.h index d27bf5db8377..3779c78203ed 100644 --- a/libc/include/llvm-libc-types/__mutex_type.h +++ b/libc/include/llvm-libc-types/__mutex_type.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES___MUTEX_TYPE_H #define LLVM_LIBC_TYPES___MUTEX_TYPE_H -#include +#include "llvm-libc-types/__futex_word.h" typedef struct { unsigned char __timed; diff --git a/libc/include/llvm-libc-types/cookie_io_functions_t.h b/libc/include/llvm-libc-types/cookie_io_functions_t.h index f9fa1a2d50ed..a3e7c32a5096 100644 --- a/libc/include/llvm-libc-types/cookie_io_functions_t.h +++ b/libc/include/llvm-libc-types/cookie_io_functions_t.h @@ -9,9 +9,9 @@ #ifndef LLVM_LIBC_TYPES_COOKIE_IO_FUNCTIONS_T_H #define LLVM_LIBC_TYPES_COOKIE_IO_FUNCTIONS_T_H -#include -#include -#include +#include "llvm-libc-types/off64_t.h" +#include "llvm-libc-types/size_t.h" +#include "llvm-libc-types/ssize_t.h" typedef ssize_t cookie_read_function_t(void *, char *, size_t); typedef ssize_t cookie_write_function_t(void *, const char *, size_t); diff --git a/libc/include/llvm-libc-types/fd_set.h b/libc/include/llvm-libc-types/fd_set.h index 58fc438bbdd2..fd1bde24c90e 100644 --- a/libc/include/llvm-libc-types/fd_set.h +++ b/libc/include/llvm-libc-types/fd_set.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES_FD_SET_H #define LLVM_LIBC_TYPES_FD_SET_H -#include // FD_SETSIZE +#include "llvm-libc-macros/sys-select-macros.h" // FD_SETSIZE typedef struct { __FD_SET_WORD_TYPE __set[__FD_SET_ARRAYSIZE]; diff --git a/libc/include/llvm-libc-types/mtx_t.h b/libc/include/llvm-libc-types/mtx_t.h index 0f3882c26b6b..ebf79871c935 100644 --- a/libc/include/llvm-libc-types/mtx_t.h +++ b/libc/include/llvm-libc-types/mtx_t.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES_MTX_T_H #define LLVM_LIBC_TYPES_MTX_T_H -#include +#include "llvm-libc-types/__mutex_type.h" typedef __mutex_type mtx_t; diff --git a/libc/include/llvm-libc-types/once_flag.h b/libc/include/llvm-libc-types/once_flag.h index cb8011284610..f80d35e317e9 100644 --- a/libc/include/llvm-libc-types/once_flag.h +++ b/libc/include/llvm-libc-types/once_flag.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES_ONCE_FLAG_H #define LLVM_LIBC_TYPES_ONCE_FLAG_H -#include +#include "llvm-libc-types/__futex_word.h" #ifdef __linux__ typedef __futex_word once_flag; diff --git a/libc/include/llvm-libc-types/pthread_attr_t.h b/libc/include/llvm-libc-types/pthread_attr_t.h index 66c04de04a99..7512193ef97b 100644 --- a/libc/include/llvm-libc-types/pthread_attr_t.h +++ b/libc/include/llvm-libc-types/pthread_attr_t.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES_PTHREAD_ATTR_T_H #define LLVM_LIBC_TYPES_PTHREAD_ATTR_T_H -#include +#include "llvm-libc-types/size_t.h" typedef struct { int __detachstate; diff --git a/libc/include/llvm-libc-types/pthread_mutex_t.h b/libc/include/llvm-libc-types/pthread_mutex_t.h index b1eb21f24fac..cf2194d719f3 100644 --- a/libc/include/llvm-libc-types/pthread_mutex_t.h +++ b/libc/include/llvm-libc-types/pthread_mutex_t.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES_PTHREAD_MUTEX_T_H #define LLVM_LIBC_TYPES_PTHREAD_MUTEX_T_H -#include +#include "llvm-libc-types/__mutex_type.h" typedef __mutex_type pthread_mutex_t; diff --git a/libc/include/llvm-libc-types/pthread_once_t.h b/libc/include/llvm-libc-types/pthread_once_t.h index 3fe78b7ddff6..8ea926f4ee7d 100644 --- a/libc/include/llvm-libc-types/pthread_once_t.h +++ b/libc/include/llvm-libc-types/pthread_once_t.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES_PTHREAD_ONCE_T_H #define LLVM_LIBC_TYPES_PTHREAD_ONCE_T_H -#include +#include "llvm-libc-types/__futex_word.h" #ifdef __linux__ typedef __futex_word pthread_once_t; diff --git a/libc/include/llvm-libc-types/pthread_t.h b/libc/include/llvm-libc-types/pthread_t.h index 72c14e1c2eea..63cc0d7dd74c 100644 --- a/libc/include/llvm-libc-types/pthread_t.h +++ b/libc/include/llvm-libc-types/pthread_t.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES_PTHREAD_T_H #define LLVM_LIBC_TYPES_PTHREAD_T_H -#include +#include "llvm-libc-types/__thread_type.h" typedef __thread_type pthread_t; diff --git a/libc/include/llvm-libc-types/siginfo_t.h b/libc/include/llvm-libc-types/siginfo_t.h index 935ef4bbcb72..dafe9c1b5f8e 100644 --- a/libc/include/llvm-libc-types/siginfo_t.h +++ b/libc/include/llvm-libc-types/siginfo_t.h @@ -9,10 +9,10 @@ #ifndef LLVM_LIBC_TYPES_SIGINFO_T_H #define LLVM_LIBC_TYPES_SIGINFO_T_H -#include -#include -#include -#include +#include "llvm-libc-types/clock_t.h" +#include "llvm-libc-types/pid_t.h" +#include "llvm-libc-types/uid_t.h" +#include "llvm-libc-types/union_sigval.h" #define SI_MAX_SIZE 128 diff --git a/libc/include/llvm-libc-types/sigset_t.h b/libc/include/llvm-libc-types/sigset_t.h index f159c6c6c643..311a92b823ff 100644 --- a/libc/include/llvm-libc-types/sigset_t.h +++ b/libc/include/llvm-libc-types/sigset_t.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES_SIGSET_T_H #define LLVM_LIBC_TYPES_SIGSET_T_H -#include +#include "llvm-libc-macros/signal-macros.h" // This definition can be adjusted/specialized for different targets and // platforms as necessary. This definition works for Linux on most targets. diff --git a/libc/include/llvm-libc-types/stack_t.h b/libc/include/llvm-libc-types/stack_t.h index 5fa4d3a6d3dc..9156425436e9 100644 --- a/libc/include/llvm-libc-types/stack_t.h +++ b/libc/include/llvm-libc-types/stack_t.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES_STACK_T_H #define LLVM_LIBC_TYPES_STACK_T_H -#include +#include "llvm-libc-types/size_t.h" typedef struct { // The order of the fields declared here should match the kernel definition diff --git a/libc/include/llvm-libc-types/struct_dirent.h b/libc/include/llvm-libc-types/struct_dirent.h index 3c5b361c3cbc..0bb71b9f3b84 100644 --- a/libc/include/llvm-libc-types/struct_dirent.h +++ b/libc/include/llvm-libc-types/struct_dirent.h @@ -9,8 +9,8 @@ #ifndef LLVM_LIBC_TYPES_STRUCT_DIRENT_H #define LLVM_LIBC_TYPES_STRUCT_DIRENT_H -#include -#include +#include "llvm-libc-types/ino_t.h" +#include "llvm-libc-types/off_t.h" struct dirent { ino_t d_ino; diff --git a/libc/include/llvm-libc-types/struct_epoll_event.h b/libc/include/llvm-libc-types/struct_epoll_event.h index 6fc5b410348a..66cf86c1e2a0 100644 --- a/libc/include/llvm-libc-types/struct_epoll_event.h +++ b/libc/include/llvm-libc-types/struct_epoll_event.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES_STRUCT_EPOLL_EVENT_H #define LLVM_LIBC_TYPES_STRUCT_EPOLL_EVENT_H -#include +#include "llvm-libc-types/struct_epoll_data.h" typedef struct epoll_event { __UINT32_TYPE__ events; diff --git a/libc/include/llvm-libc-types/struct_rlimit.h b/libc/include/llvm-libc-types/struct_rlimit.h index e093d9f306c9..11e6bee15f9d 100644 --- a/libc/include/llvm-libc-types/struct_rlimit.h +++ b/libc/include/llvm-libc-types/struct_rlimit.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES_STRUCT_RLIMIT_H #define LLVM_LIBC_TYPES_STRUCT_RLIMIT_H -#include +#include "llvm-libc-types/rlim_t.h" struct rlimit { rlim_t rlim_cur; diff --git a/libc/include/llvm-libc-types/struct_rusage.h b/libc/include/llvm-libc-types/struct_rusage.h index 21ea8b1061c2..ed838d30ede3 100644 --- a/libc/include/llvm-libc-types/struct_rusage.h +++ b/libc/include/llvm-libc-types/struct_rusage.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES_STRUCT_RUSAGE_H #define LLVM_LIBC_TYPES_STRUCT_RUSAGE_H -#include +#include "llvm-libc-types/struct_timeval.h" struct rusage { struct timeval ru_utime; diff --git a/libc/include/llvm-libc-types/struct_sched_param.h b/libc/include/llvm-libc-types/struct_sched_param.h index 0521a4df652f..86209ac3a181 100644 --- a/libc/include/llvm-libc-types/struct_sched_param.h +++ b/libc/include/llvm-libc-types/struct_sched_param.h @@ -9,9 +9,9 @@ #ifndef LLVM_LIBC_TYPES_STRUCT_SCHED_PARAM_H #define LLVM_LIBC_TYPES_STRUCT_SCHED_PARAM_H -#include -#include -#include +#include "llvm-libc-types/pid_t.h" +#include "llvm-libc-types/struct_timespec.h" +#include "llvm-libc-types/time_t.h" struct sched_param { // Process or thread execution scheduling priority. diff --git a/libc/include/llvm-libc-types/struct_sigaction.h b/libc/include/llvm-libc-types/struct_sigaction.h index 54d2995f4ecd..ffce04d0f7e8 100644 --- a/libc/include/llvm-libc-types/struct_sigaction.h +++ b/libc/include/llvm-libc-types/struct_sigaction.h @@ -9,8 +9,8 @@ #ifndef LLVM_LIBC_TYPES_STRUCT_SIGACTION_H #define LLVM_LIBC_TYPES_STRUCT_SIGACTION_H -#include -#include +#include "llvm-libc-types/siginfo_t.h" +#include "llvm-libc-types/sigset_t.h" struct sigaction { union { diff --git a/libc/include/llvm-libc-types/struct_sockaddr.h b/libc/include/llvm-libc-types/struct_sockaddr.h index 074b1ae50ef0..a98606323c52 100644 --- a/libc/include/llvm-libc-types/struct_sockaddr.h +++ b/libc/include/llvm-libc-types/struct_sockaddr.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES_STRUCT_SOCKADDR_H #define LLVM_LIBC_TYPES_STRUCT_SOCKADDR_H -#include +#include "llvm-libc-types/sa_family_t.h" struct sockaddr { sa_family_t sa_family; diff --git a/libc/include/llvm-libc-types/struct_sockaddr_un.h b/libc/include/llvm-libc-types/struct_sockaddr_un.h index 4332419a5b71..3c0362ce24fb 100644 --- a/libc/include/llvm-libc-types/struct_sockaddr_un.h +++ b/libc/include/llvm-libc-types/struct_sockaddr_un.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES_STRUCT_SOCKADDR_UN_H #define LLVM_LIBC_TYPES_STRUCT_SOCKADDR_UN_H -#include +#include "llvm-libc-types/sa_family_t.h" // This is the sockaddr specialization for AF_UNIX or AF_LOCAL sockets, as // defined by posix. diff --git a/libc/include/llvm-libc-types/struct_stat.h b/libc/include/llvm-libc-types/struct_stat.h index 3539fb5b920e..d8ae9dd6ffdc 100644 --- a/libc/include/llvm-libc-types/struct_stat.h +++ b/libc/include/llvm-libc-types/struct_stat.h @@ -9,16 +9,16 @@ #ifndef LLVM_LIBC_TYPES_STRUCT_STAT_H #define LLVM_LIBC_TYPES_STRUCT_STAT_H -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include "llvm-libc-types/blkcnt_t.h" +#include "llvm-libc-types/blksize_t.h" +#include "llvm-libc-types/dev_t.h" +#include "llvm-libc-types/gid_t.h" +#include "llvm-libc-types/ino_t.h" +#include "llvm-libc-types/mode_t.h" +#include "llvm-libc-types/nlink_t.h" +#include "llvm-libc-types/off_t.h" +#include "llvm-libc-types/struct_timespec.h" +#include "llvm-libc-types/uid_t.h" struct stat { dev_t st_dev; diff --git a/libc/include/llvm-libc-types/struct_termios.h b/libc/include/llvm-libc-types/struct_termios.h index 72aefe4f6926..51241192f741 100644 --- a/libc/include/llvm-libc-types/struct_termios.h +++ b/libc/include/llvm-libc-types/struct_termios.h @@ -9,9 +9,9 @@ #ifndef __LLVM_LIBC_TYPES_STRUCT_TERMIOS_H__ #define __LLVM_LIBC_TYPES_STRUCT_TERMIOS_H__ -#include -#include -#include +#include "llvm-libc-types/cc_t.h" +#include "llvm-libc-types/speed_t.h" +#include "llvm-libc-types/tcflag_t.h" struct termios { tcflag_t c_iflag; // Input mode flags diff --git a/libc/include/llvm-libc-types/struct_timespec.h b/libc/include/llvm-libc-types/struct_timespec.h index 5d56d9c9468b..4baab07c10f8 100644 --- a/libc/include/llvm-libc-types/struct_timespec.h +++ b/libc/include/llvm-libc-types/struct_timespec.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES_STRUCT_TIMESPEC_H #define LLVM_LIBC_TYPES_STRUCT_TIMESPEC_H -#include +#include "llvm-libc-types/time_t.h" struct timespec { time_t tv_sec; /* Seconds. */ diff --git a/libc/include/llvm-libc-types/struct_timeval.h b/libc/include/llvm-libc-types/struct_timeval.h index 6a0b7bbaf825..365b835d345d 100644 --- a/libc/include/llvm-libc-types/struct_timeval.h +++ b/libc/include/llvm-libc-types/struct_timeval.h @@ -9,8 +9,8 @@ #ifndef LLVM_LIBC_TYPES_STRUCT_TIMEVAL_H #define LLVM_LIBC_TYPES_STRUCT_TIMEVAL_H -#include -#include +#include "llvm-libc-types/suseconds_t.h" +#include "llvm-libc-types/time_t.h" struct timeval { time_t tv_sec; // Seconds diff --git a/libc/include/llvm-libc-types/thrd_t.h b/libc/include/llvm-libc-types/thrd_t.h index 2e0f9a0d75ad..751ea5b9e4c0 100644 --- a/libc/include/llvm-libc-types/thrd_t.h +++ b/libc/include/llvm-libc-types/thrd_t.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES_THRD_T_H #define LLVM_LIBC_TYPES_THRD_T_H -#include +#include "llvm-libc-types/__thread_type.h" typedef __thread_type thrd_t; diff --git a/libc/include/math.h.def b/libc/include/math.h.def index 927e2d6697c6..cd2fe76f40bf 100644 --- a/libc/include/math.h.def +++ b/libc/include/math.h.def @@ -9,9 +9,9 @@ #ifndef LLVM_LIBC_MATH_H #define LLVM_LIBC_MATH_H -#include <__llvm-libc-common.h> -#include -#include +#include "__llvm-libc-common.h" +#include "llvm-libc-macros/math-macros.h" +#include "llvm-libc-types/float128.h" %%public_api() diff --git a/libc/include/pthread.h.def b/libc/include/pthread.h.def index 391ecd3c124f..abeb839ee83d 100644 --- a/libc/include/pthread.h.def +++ b/libc/include/pthread.h.def @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_PTHREAD_H #define LLVM_LIBC_PTHREAD_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" #define PTHREAD_STACK_MIN (1 << 14) // 16KB diff --git a/libc/include/sched.h.def b/libc/include/sched.h.def index 3b2d5e330859..493028e8dcc4 100644 --- a/libc/include/sched.h.def +++ b/libc/include/sched.h.def @@ -9,8 +9,8 @@ #ifndef LLVM_LIBC_SCHED_H #define LLVM_LIBC_SCHED_H -#include <__llvm-libc-common.h> -#include +#include "__llvm-libc-common.h" +#include "llvm-libc-macros/sched-macros.h" %%public_api() diff --git a/libc/include/search.h.def b/libc/include/search.h.def index 3435c1f8ad04..6301ba7b656c 100644 --- a/libc/include/search.h.def +++ b/libc/include/search.h.def @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_SEARCH_H #define LLVM_LIBC_SEARCH_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" #define __need_size_t #include diff --git a/libc/include/setjmp.h.def b/libc/include/setjmp.h.def index 7447be2415bd..670bc1ac0fe2 100644 --- a/libc/include/setjmp.h.def +++ b/libc/include/setjmp.h.def @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_SETJMP_H #define LLVM_LIBC_SETJMP_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" %%public_api() diff --git a/libc/include/signal.h.def b/libc/include/signal.h.def index 0e7033452715..50a5f44c7337 100644 --- a/libc/include/signal.h.def +++ b/libc/include/signal.h.def @@ -9,12 +9,12 @@ #ifndef LLVM_LIBC_SIGNAL_H #define LLVM_LIBC_SIGNAL_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" #define __need_size_t #include -#include +#include "llvm-libc-macros/signal-macros.h" %%public_api() diff --git a/libc/include/spawn.h.def b/libc/include/spawn.h.def index 368ebff17ca1..a8d701585286 100644 --- a/libc/include/spawn.h.def +++ b/libc/include/spawn.h.def @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_SPAWN_H #define LLVM_LIBC_SPAWN_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" %%public_api() diff --git a/libc/include/stdbit.h.def b/libc/include/stdbit.h.def index c5a77329fbfe..28c147b01e22 100644 --- a/libc/include/stdbit.h.def +++ b/libc/include/stdbit.h.def @@ -9,10 +9,10 @@ #ifndef LLVM_LIBC_STDBIT_H #define LLVM_LIBC_STDBIT_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" %%public_api() -#include +#include "llvm-libc-macros/stdbit-macros.h" #endif // LLVM_LIBC_STDBIT_H diff --git a/libc/include/stdckdint.h.def b/libc/include/stdckdint.h.def index c82470911c33..d4a9d829a3c9 100644 --- a/libc/include/stdckdint.h.def +++ b/libc/include/stdckdint.h.def @@ -9,10 +9,10 @@ #ifndef LLVM_LIBC_STDCKDINT_H #define LLVM_LIBC_STDCKDINT_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" %%public_api() -#include +#include "llvm-libc-macros/stdckdint-macros.h" #endif // LLVM_LIBC_STDCKDINT_H diff --git a/libc/include/stdfix.h.def b/libc/include/stdfix.h.def index 368eeb33f2f0..8ac49be45fba 100644 --- a/libc/include/stdfix.h.def +++ b/libc/include/stdfix.h.def @@ -9,8 +9,8 @@ #ifndef LLVM_LIBC_STDFIX_H #define LLVM_LIBC_STDFIX_H -#include <__llvm-libc-common.h> -#include +#include "__llvm-libc-common.h" +#include "llvm-libc-macros/stdfix-macros.h" // From ISO/IEC TR 18037:2008 standard: // https://www.iso.org/standard/51126.html diff --git a/libc/include/stdint.h.def b/libc/include/stdint.h.def index 9e269101acd2..d7660860c91f 100644 --- a/libc/include/stdint.h.def +++ b/libc/include/stdint.h.def @@ -9,6 +9,6 @@ #ifndef LLVM_LIBC_STDINT_H #define LLVM_LIBC_STDINT_H -#include +#include "llvm-libc-macros/stdint-macros.h" #endif // LLVM_LIBC_STDINT_H diff --git a/libc/include/stdio.h.def b/libc/include/stdio.h.def index a28d009288d5..78d800c83b51 100644 --- a/libc/include/stdio.h.def +++ b/libc/include/stdio.h.def @@ -9,9 +9,9 @@ #ifndef LLVM_LIBC_STDIO_H #define LLVM_LIBC_STDIO_H -#include <__llvm-libc-common.h> -#include -#include +#include "__llvm-libc-common.h" +#include "llvm-libc-macros/file-seek-macros.h" +#include "llvm-libc-macros/stdio-macros.h" #include diff --git a/libc/include/stdlib.h.def b/libc/include/stdlib.h.def index 18df71a49a9b..d523f7a53024 100644 --- a/libc/include/stdlib.h.def +++ b/libc/include/stdlib.h.def @@ -9,8 +9,8 @@ #ifndef LLVM_LIBC_STDLIB_H #define LLVM_LIBC_STDLIB_H -#include <__llvm-libc-common.h> -#include +#include "__llvm-libc-common.h" +#include "llvm-libc-macros/stdlib-macros.h" %%public_api() diff --git a/libc/include/string.h.def b/libc/include/string.h.def index 26e6ef93d314..1bd2687db2be 100644 --- a/libc/include/string.h.def +++ b/libc/include/string.h.def @@ -9,9 +9,9 @@ #ifndef LLVM_LIBC_STRING_H #define LLVM_LIBC_STRING_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" -#include +#include "llvm-libc-macros/null-macro.h" %%public_api() diff --git a/libc/include/strings.h.def b/libc/include/strings.h.def index f07ca30d5dbd..9b016bf0bc50 100644 --- a/libc/include/strings.h.def +++ b/libc/include/strings.h.def @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_STRINGS_H #define LLVM_LIBC_STRINGS_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" %%public_api() diff --git a/libc/include/sys/auxv.h.def b/libc/include/sys/auxv.h.def index 504c2f68cb1e..11ab25bcfe2c 100644 --- a/libc/include/sys/auxv.h.def +++ b/libc/include/sys/auxv.h.def @@ -9,9 +9,9 @@ #ifndef LLVM_LIBC_SYS_AUXV_H #define LLVM_LIBC_SYS_AUXV_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" -#include +#include "llvm-libc-macros/sys-auxv-macros.h" %%public_api() diff --git a/libc/include/sys/epoll.h.def b/libc/include/sys/epoll.h.def index 490fad91db3c..85f7d9ad6091 100644 --- a/libc/include/sys/epoll.h.def +++ b/libc/include/sys/epoll.h.def @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_SYS_EPOLL_H #define LLVM_LIBC_SYS_EPOLL_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" %%public_api() diff --git a/libc/include/sys/ioctl.h.def b/libc/include/sys/ioctl.h.def index 90d91cf38291..2f37a1190ac1 100644 --- a/libc/include/sys/ioctl.h.def +++ b/libc/include/sys/ioctl.h.def @@ -9,9 +9,9 @@ #ifndef LLVM_LIBC_SYS_IOCTL_H #define LLVM_LIBC_SYS_IOCTL_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" -#include +#include "llvm-libc-macros/sys-ioctl-macros.h" %%public_api() diff --git a/libc/include/sys/mman.h.def b/libc/include/sys/mman.h.def index ab9fde1bb920..2e2c2f1997b8 100644 --- a/libc/include/sys/mman.h.def +++ b/libc/include/sys/mman.h.def @@ -9,9 +9,9 @@ #ifndef LLVM_LIBC_SYS_MMAN_H #define LLVM_LIBC_SYS_MMAN_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" -#include +#include "llvm-libc-macros/sys-mman-macros.h" %%public_api() diff --git a/libc/include/sys/prctl.h.def b/libc/include/sys/prctl.h.def index 0a11543d0729..08648c9f4792 100644 --- a/libc/include/sys/prctl.h.def +++ b/libc/include/sys/prctl.h.def @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_SYS_PRCTL_H #define LLVM_LIBC_SYS_PRCTL_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" // Process control is highly platform specific, so the platform usually defines // the macros itself. diff --git a/libc/include/sys/queue.h b/libc/include/sys/queue.h index 1cde35e77a04..cca53c16f0f3 100644 --- a/libc/include/sys/queue.h +++ b/libc/include/sys/queue.h @@ -9,6 +9,6 @@ #ifndef SYS_QUEUE_H #define SYS_QUEUE_H -#include +#include "llvm-libc-macros/sys-queue-macros.h" #endif // SYS_QUEUE_H diff --git a/libc/include/sys/random.h.def b/libc/include/sys/random.h.def index b767f2479fc2..d11431b2755d 100644 --- a/libc/include/sys/random.h.def +++ b/libc/include/sys/random.h.def @@ -9,9 +9,9 @@ #ifndef LLVM_LIBC_SYS_RANDOM_H #define LLVM_LIBC_SYS_RANDOM_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" -#include +#include "llvm-libc-macros/sys-random-macros.h" %%public_api() diff --git a/libc/include/sys/resource.h.def b/libc/include/sys/resource.h.def index 31132d3b2608..365d803cf715 100644 --- a/libc/include/sys/resource.h.def +++ b/libc/include/sys/resource.h.def @@ -9,9 +9,9 @@ #ifndef LLVM_LIBC_SYS_RESOURCE_H #define LLVM_LIBC_SYS_RESOURCE_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" -#include +#include "llvm-libc-macros/sys-resource-macros.h" %%public_api() diff --git a/libc/include/sys/select.h.def b/libc/include/sys/select.h.def index 4f3cebaecbb9..529be7158f26 100644 --- a/libc/include/sys/select.h.def +++ b/libc/include/sys/select.h.def @@ -9,9 +9,9 @@ #ifndef LLVM_LIBC_SYS_SELECT_H #define LLVM_LIBC_SYS_SELECT_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" -#include +#include "llvm-libc-macros/sys-select-macros.h" %%public_api() diff --git a/libc/include/sys/sendfile.h.def b/libc/include/sys/sendfile.h.def index 947edc28e1ec..d7f21f91f95e 100644 --- a/libc/include/sys/sendfile.h.def +++ b/libc/include/sys/sendfile.h.def @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_SYS_SENDFILE_H #define LLVM_LIBC_SYS_SENDFILE_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" %%public_api() diff --git a/libc/include/sys/socket.h.def b/libc/include/sys/socket.h.def index 71654c64b988..933ef1512e45 100644 --- a/libc/include/sys/socket.h.def +++ b/libc/include/sys/socket.h.def @@ -9,9 +9,9 @@ #ifndef LLVM_LIBC_SYS_SOCKET_H #define LLVM_LIBC_SYS_SOCKET_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" -#include +#include "llvm-libc-macros/sys-socket-macros.h" %%public_api() diff --git a/libc/include/sys/stat.h.def b/libc/include/sys/stat.h.def index ed37d010f497..06a98a4aa029 100644 --- a/libc/include/sys/stat.h.def +++ b/libc/include/sys/stat.h.def @@ -9,9 +9,9 @@ #ifndef LLVM_LIBC_SYS_STAT_H #define LLVM_LIBC_SYS_STAT_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" -#include +#include "llvm-libc-macros/sys-stat-macros.h" %%public_api() diff --git a/libc/include/sys/time.h.def b/libc/include/sys/time.h.def index 9a3bd7bb49f8..5a87139aefc9 100644 --- a/libc/include/sys/time.h.def +++ b/libc/include/sys/time.h.def @@ -9,11 +9,11 @@ #ifndef LLVM_LIBC_SYS_TIME_H #define LLVM_LIBC_SYS_TIME_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" -#include +#include "llvm-libc-types/struct_timeval.h" -#include +#include "llvm-libc-macros/sys-time-macros.h" %%public_api() diff --git a/libc/include/sys/types.h.def b/libc/include/sys/types.h.def index 689482973fc7..f5c3bb2c928b 100644 --- a/libc/include/sys/types.h.def +++ b/libc/include/sys/types.h.def @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_SYS_TYPES_H #define LLVM_LIBC_SYS_TYPES_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" %%public_api() diff --git a/libc/include/sys/utsname.h.def b/libc/include/sys/utsname.h.def index 6d7daeb45f01..08dbbfc06245 100644 --- a/libc/include/sys/utsname.h.def +++ b/libc/include/sys/utsname.h.def @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_SYS_UTSNAME_H #define LLVM_LIBC_SYS_UTSNAME_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" %%public_api() diff --git a/libc/include/sys/wait.h.def b/libc/include/sys/wait.h.def index b4fcce4d1652..0a76da019fdc 100644 --- a/libc/include/sys/wait.h.def +++ b/libc/include/sys/wait.h.def @@ -9,9 +9,9 @@ #ifndef LLVM_LIBC_SYS_WAIT_H #define LLVM_LIBC_SYS_WAIT_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" -#include +#include "llvm-libc-macros/sys-wait-macros.h" %%public_api() diff --git a/libc/include/termios.h.def b/libc/include/termios.h.def index be1cd2bff526..7538944c0985 100644 --- a/libc/include/termios.h.def +++ b/libc/include/termios.h.def @@ -9,8 +9,8 @@ #ifndef LLVM_LIBC_TERMIOS_H #define LLVM_LIBC_TERMIOS_H -#include <__llvm-libc-common.h> -#include +#include "__llvm-libc-common.h" +#include "llvm-libc-macros/termios-macros.h" %%public_api() diff --git a/libc/include/threads.h.def b/libc/include/threads.h.def index 93541b8d3bac..b114bea0ace3 100644 --- a/libc/include/threads.h.def +++ b/libc/include/threads.h.def @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_THREADS_H #define LLVM_LIBC_THREADS_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" %%public_api() diff --git a/libc/include/time.h.def b/libc/include/time.h.def index d8988329a372..2355e8822fad 100644 --- a/libc/include/time.h.def +++ b/libc/include/time.h.def @@ -9,8 +9,8 @@ #ifndef LLVM_LIBC_TIME_H #define LLVM_LIBC_TIME_H -#include <__llvm-libc-common.h> -#include +#include "__llvm-libc-common.h" +#include "llvm-libc-macros/time-macros.h" %%public_api() diff --git a/libc/include/uchar.h.def b/libc/include/uchar.h.def index 7e62d43e9cc4..31b7fcb73ded 100644 --- a/libc/include/uchar.h.def +++ b/libc/include/uchar.h.def @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_UCHAR_H #define LLVM_LIBC_UCHAR_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" %%public_api() diff --git a/libc/include/unistd.h.def b/libc/include/unistd.h.def index fa10af653fae..6b9137e14623 100644 --- a/libc/include/unistd.h.def +++ b/libc/include/unistd.h.def @@ -9,9 +9,9 @@ #ifndef LLVM_LIBC_UNISTD_H #define LLVM_LIBC_UNISTD_H -#include <__llvm-libc-common.h> -#include -#include +#include "__llvm-libc-common.h" +#include "llvm-libc-macros/file-seek-macros.h" +#include "llvm-libc-macros/unistd-macros.h" %%public_api() diff --git a/libc/include/wchar.h.def b/libc/include/wchar.h.def index ac72f80aa083..4c25de700d60 100644 --- a/libc/include/wchar.h.def +++ b/libc/include/wchar.h.def @@ -9,8 +9,8 @@ #ifndef LLVM_LIBC_WCHAR_H #define LLVM_LIBC_WCHAR_H -#include <__llvm-libc-common.h> -#include +#include "__llvm-libc-common.h" +#include "llvm-libc-macros/wchar-macros.h" %%public_api() -- GitLab From 4318f7e5301fb737a7abaacb3b43b6a9289055f3 Mon Sep 17 00:00:00 2001 From: Nick Desaulniers Date: Fri, 22 Mar 2024 08:28:03 -0700 Subject: [PATCH 065/306] [libc][stdlib] initial support for __cxa_finalize (#85865) I'm trying to break up the pieces of supporting __cxa_finalize into smaller commits. Provide this symbol first, and make use of it from exit. Next will be to store __dso_handle, then finally to only run callbacks that were registered from a specific dso. Link: #85651 Link: https://itanium-cxx-abi.github.io/cxx-abi/abi.html#dso-dtor --- libc/src/stdlib/atexit.cpp | 31 +++++++++++++++++++------------ libc/src/stdlib/exit.cpp | 8 +++----- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/libc/src/stdlib/atexit.cpp b/libc/src/stdlib/atexit.cpp index 741ea4f25103..fa072b2fdf8d 100644 --- a/libc/src/stdlib/atexit.cpp +++ b/libc/src/stdlib/atexit.cpp @@ -55,14 +55,10 @@ void stdc_at_exit_func(void *payload) { reinterpret_cast(payload)(); } -} // namespace - -namespace internal { - void call_exit_callbacks() { handler_list_mtx.lock(); while (!exit_callbacks.empty()) { - auto unit = exit_callbacks.back(); + AtExitUnit &unit = exit_callbacks.back(); exit_callbacks.pop_back(); handler_list_mtx.unlock(); unit.callback(unit.payload); @@ -71,20 +67,31 @@ void call_exit_callbacks() { ExitCallbackList::destroy(&exit_callbacks); } -} // namespace internal - -static int add_atexit_unit(const AtExitUnit &unit) { +int add_atexit_unit(const AtExitUnit &unit) { MutexLock lock(&handler_list_mtx); - if (!exit_callbacks.push_back(unit)) - return -1; - return 0; + if (exit_callbacks.push_back(unit)) + return 0; + return -1; } +} // namespace + +extern "C" { + // TODO: Handle the last dso handle argument. -extern "C" int __cxa_atexit(AtExitCallback *callback, void *payload, void *) { +int __cxa_atexit(AtExitCallback *callback, void *payload, void *) { return add_atexit_unit({callback, payload}); } +// TODO: Handle the dso handle argument. call_exit_callbacks should only invoke +// the callbacks from this DSO. Requires adding support for __dso_handle. +void __cxa_finalize(void *dso) { + if (!dso) + call_exit_callbacks(); +} + +} // extern "C" + LLVM_LIBC_FUNCTION(int, atexit, (StdCAtExitCallback * callback)) { return add_atexit_unit( {&stdc_at_exit_func, reinterpret_cast(callback)}); diff --git a/libc/src/stdlib/exit.cpp b/libc/src/stdlib/exit.cpp index cc5ae6648d11..e754b34e4698 100644 --- a/libc/src/stdlib/exit.cpp +++ b/libc/src/stdlib/exit.cpp @@ -10,14 +10,12 @@ #include "src/__support/OSUtil/quick_exit.h" #include "src/__support/common.h" -namespace LIBC_NAMESPACE { +extern "C" void __cxa_finalize(void *); -namespace internal { -void call_exit_callbacks(); -} +namespace LIBC_NAMESPACE { LLVM_LIBC_FUNCTION(void, exit, (int status)) { - internal::call_exit_callbacks(); + __cxa_finalize(nullptr); quick_exit(status); __builtin_unreachable(); } -- GitLab From 6e28ecd79995a72a8dbde8f16a1afc18309442a1 Mon Sep 17 00:00:00 2001 From: Antonio Frighetto Date: Fri, 22 Mar 2024 16:23:19 +0100 Subject: [PATCH 066/306] [Object][ELF] Ensure offset to locate dyn section does not go past size Validate `p_offset` in `dynamicEntries` before computing the entry offset. Fixes: https://github.com/llvm/llvm-project/issues/85568. --- llvm/lib/Object/ELF.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/llvm/lib/Object/ELF.cpp b/llvm/lib/Object/ELF.cpp index 55dd0c8e06c0..0ac4e7a57759 100644 --- a/llvm/lib/Object/ELF.cpp +++ b/llvm/lib/Object/ELF.cpp @@ -560,7 +560,11 @@ Expected ELFFile::dynamicEntries() const { for (const Elf_Phdr &Phdr : *ProgramHeadersOrError) { if (Phdr.p_type == ELF::PT_DYNAMIC) { - Dyn = ArrayRef(reinterpret_cast(base() + Phdr.p_offset), + const uint8_t *DynOffset = base() + Phdr.p_offset; + if (DynOffset > end()) + return createError( + "dynamic section offset past file size: corrupted ELF"); + Dyn = ArrayRef(reinterpret_cast(DynOffset), Phdr.p_filesz / sizeof(Elf_Dyn)); break; } -- GitLab From f66d631bf8dc0fe33c6ba88c3dc7f00ac5946065 Mon Sep 17 00:00:00 2001 From: Amir Ayupov Date: Fri, 22 Mar 2024 08:38:40 -0700 Subject: [PATCH 067/306] Revert "[BOLT] Add BB index to BAT (#86044)" This reverts commit 3b3de48fd84b8269d5f45ee0a9dc6b7448368424. --- bolt/docs/BAT.md | 11 +++-- .../bolt/Profile/BoltAddressTranslation.h | 7 +--- bolt/lib/Profile/BoltAddressTranslation.cpp | 39 +++++------------ .../X86/bolt-address-translation-yaml.test | 2 +- bolt/test/X86/bolt-address-translation.test | 2 +- clang/lib/Driver/ToolChains/Clang.cpp | 4 +- clang/test/Driver/unsupported-option-gpu.c | 1 + lld/MachO/Driver.cpp | 42 ++----------------- lld/MachO/InputSection.cpp | 38 +++++++++++++++++ lld/MachO/InputSection.h | 3 ++ lld/MachO/ObjC.cpp | 16 +++---- lld/MachO/SyntheticSections.cpp | 4 +- 12 files changed, 77 insertions(+), 92 deletions(-) diff --git a/bolt/docs/BAT.md b/bolt/docs/BAT.md index 436593478a39..186b0e5ea89d 100644 --- a/bolt/docs/BAT.md +++ b/bolt/docs/BAT.md @@ -90,12 +90,11 @@ current function. ### Address translation table Delta encoding means that only the difference with the previous corresponding entry is encoded. Input offsets implicitly start at zero. -| Entry | Encoding | Description | Branch/BB | -| ------ | ------| ----------- | ------ | -| `OutputOffset` | Continuous, Delta, ULEB128 | Function offset in output binary | Both | -| `InputOffset` | Optional, Delta, SLEB128 | Function offset in input binary with `BRANCHENTRY` LSB bit | Both | -| `BBHash` | Optional, 8b | Basic block hash in input binary | BB | -| `BBIdx` | Optional, Delta, ULEB128 | Basic block index in input binary | BB | +| Entry | Encoding | Description | +| ------ | ------| ----------- | +| `OutputOffset` | Continuous, Delta, ULEB128 | Function offset in output binary | +| `InputOffset` | Optional, Delta, SLEB128 | Function offset in input binary with `BRANCHENTRY` LSB bit | +| `BBHash` | Optional, 8b | Basic block entries only: basic block hash in input binary | `BRANCHENTRY` bit denotes whether a given offset pair is a control flow source (branch or call instruction). If not set, it signifies a control flow target diff --git a/bolt/include/bolt/Profile/BoltAddressTranslation.h b/bolt/include/bolt/Profile/BoltAddressTranslation.h index eda2b318f0d0..1f53f6d344ad 100644 --- a/bolt/include/bolt/Profile/BoltAddressTranslation.h +++ b/bolt/include/bolt/Profile/BoltAddressTranslation.h @@ -122,10 +122,6 @@ public: /// Returns BF hash by function output address (after BOLT). size_t getBFHash(uint64_t OutputAddress) const; - /// Returns BB index by function output address (after BOLT) and basic block - /// input offset. - unsigned getBBIndex(uint64_t FuncOutputAddress, uint32_t BBInputOffset) const; - /// True if a given \p Address is a function with translation table entry. bool isBATFunction(uint64_t Address) const { return Maps.count(Address); } @@ -158,8 +154,7 @@ private: std::map Maps; - /// Map basic block input offset to a basic block index and hash pair. - using BBHashMap = std::unordered_map>; + using BBHashMap = std::unordered_map; std::unordered_map> FuncHashes; /// Links outlined cold bocks to their original function diff --git a/bolt/lib/Profile/BoltAddressTranslation.cpp b/bolt/lib/Profile/BoltAddressTranslation.cpp index 8fe976cc00e5..1d61a1b735b4 100644 --- a/bolt/lib/Profile/BoltAddressTranslation.cpp +++ b/bolt/lib/Profile/BoltAddressTranslation.cpp @@ -45,8 +45,6 @@ void BoltAddressTranslation::writeEntriesForBB(MapTy &Map, LLVM_DEBUG(dbgs() << formatv(" Hash: {0:x}\n", getBBHash(HotFuncAddress, BBInputOffset))); (void)HotFuncAddress; - LLVM_DEBUG(dbgs() << formatv(" Index: {0}\n", - getBBIndex(HotFuncAddress, BBInputOffset))); // In case of conflicts (same Key mapping to different Vals), the last // update takes precedence. Of course it is not ideal to have conflicts and // those happen when we have an empty BB that either contained only @@ -219,7 +217,6 @@ void BoltAddressTranslation::writeMaps(std::map &Maps, } size_t Index = 0; uint64_t InOffset = 0; - size_t PrevBBIndex = 0; // Output and Input addresses and delta-encoded for (std::pair &KeyVal : Map) { const uint64_t OutputAddress = KeyVal.first + Address; @@ -229,15 +226,11 @@ void BoltAddressTranslation::writeMaps(std::map &Maps, encodeSLEB128(KeyVal.second - InOffset, OS); InOffset = KeyVal.second; // Keeping InOffset as if BRANCHENTRY is encoded if ((InOffset & BRANCHENTRY) == 0) { - unsigned BBIndex; - size_t BBHash; - std::tie(BBIndex, BBHash) = FuncHashPair.second[InOffset >> 1]; + // Basic block hash + size_t BBHash = FuncHashPair.second[InOffset >> 1]; OS.write(reinterpret_cast(&BBHash), 8); - // Basic block index in the input binary - encodeULEB128(BBIndex - PrevBBIndex, OS); - PrevBBIndex = BBIndex; - LLVM_DEBUG(dbgs() << formatv("{0:x} -> {1:x} {2:x} {3}\n", KeyVal.first, - InOffset >> 1, BBHash, BBIndex)); + LLVM_DEBUG(dbgs() << formatv("{0:x} -> {1:x} {2:x}\n", KeyVal.first, + InOffset >> 1, BBHash)); } } } @@ -323,7 +316,6 @@ void BoltAddressTranslation::parseMaps(std::vector &HotFuncs, LLVM_DEBUG(dbgs() << "Parsing " << NumEntries << " entries for 0x" << Twine::utohexstr(Address) << "\n"); uint64_t InputOffset = 0; - size_t BBIndex = 0; for (uint32_t J = 0; J < NumEntries; ++J) { const uint64_t OutputDelta = DE.getULEB128(&Offset, &Err); const uint64_t OutputAddress = PrevAddress + OutputDelta; @@ -338,25 +330,19 @@ void BoltAddressTranslation::parseMaps(std::vector &HotFuncs, } Map.insert(std::pair(OutputOffset, InputOffset)); size_t BBHash = 0; - size_t BBIndexDelta = 0; const bool IsBranchEntry = InputOffset & BRANCHENTRY; if (!IsBranchEntry) { BBHash = DE.getU64(&Offset, &Err); - BBIndexDelta = DE.getULEB128(&Offset, &Err); - BBIndex += BBIndexDelta; // Map basic block hash to hot fragment by input offset - FuncHashes[HotAddress].second.emplace(InputOffset >> 1, - std::pair(BBIndex, BBHash)); + FuncHashes[HotAddress].second.emplace(InputOffset >> 1, BBHash); } LLVM_DEBUG({ dbgs() << formatv( "{0:x} -> {1:x} ({2}/{3}b -> {4}/{5}b), {6:x}", OutputOffset, InputOffset, OutputDelta, getULEB128Size(OutputDelta), InputDelta, (J < EqualElems) ? 0 : getSLEB128Size(InputDelta), OutputAddress); - if (!IsBranchEntry) { - dbgs() << formatv(" {0:x} {1}/{2}b", BBHash, BBIndex, - getULEB128Size(BBIndexDelta)); - } + if (BBHash) + dbgs() << formatv(" {0:x}", BBHash); dbgs() << '\n'; }); } @@ -508,19 +494,14 @@ void BoltAddressTranslation::saveMetadata(BinaryContext &BC) { FuncHashes[BF.getAddress()].first = BF.computeHash(); BF.computeBlockHashes(); for (const BinaryBasicBlock &BB : BF) - FuncHashes[BF.getAddress()].second.emplace( - BB.getInputOffset(), std::pair(BB.getIndex(), BB.getHash())); + FuncHashes[BF.getAddress()].second.emplace(BB.getInputOffset(), + BB.getHash()); } } -unsigned BoltAddressTranslation::getBBIndex(uint64_t FuncOutputAddress, - uint32_t BBInputOffset) const { - return FuncHashes.at(FuncOutputAddress).second.at(BBInputOffset).first; -} - size_t BoltAddressTranslation::getBBHash(uint64_t FuncOutputAddress, uint32_t BBInputOffset) const { - return FuncHashes.at(FuncOutputAddress).second.at(BBInputOffset).second; + return FuncHashes.at(FuncOutputAddress).second.at(BBInputOffset); } size_t BoltAddressTranslation::getBFHash(uint64_t OutputAddress) const { diff --git a/bolt/test/X86/bolt-address-translation-yaml.test b/bolt/test/X86/bolt-address-translation-yaml.test index 4516a662697a..25ff4e7fbfcc 100644 --- a/bolt/test/X86/bolt-address-translation-yaml.test +++ b/bolt/test/X86/bolt-address-translation-yaml.test @@ -18,7 +18,7 @@ RUN: | FileCheck --check-prefix CHECK-BOLT-YAML %s WRITE-BAT-CHECK: BOLT-INFO: Wrote 5 BAT maps WRITE-BAT-CHECK: BOLT-INFO: Wrote 4 function and 22 basic block hashes -WRITE-BAT-CHECK: BOLT-INFO: BAT section size (bytes): 376 +WRITE-BAT-CHECK: BOLT-INFO: BAT section size (bytes): 344 READ-BAT-CHECK-NOT: BOLT-ERROR: unable to save profile in YAML format for input file processed by BOLT READ-BAT-CHECK: BOLT-INFO: Parsed 5 BAT entries diff --git a/bolt/test/X86/bolt-address-translation.test b/bolt/test/X86/bolt-address-translation.test index 5c1db89e3c6b..4277b4e0d0fe 100644 --- a/bolt/test/X86/bolt-address-translation.test +++ b/bolt/test/X86/bolt-address-translation.test @@ -37,7 +37,7 @@ # CHECK: BOLT: 3 out of 7 functions were overwritten. # CHECK: BOLT-INFO: Wrote 6 BAT maps # CHECK: BOLT-INFO: Wrote 3 function and 58 basic block hashes -# CHECK: BOLT-INFO: BAT section size (bytes): 920 +# CHECK: BOLT-INFO: BAT section size (bytes): 816 # # usqrt mappings (hot part). We match against any key (left side containing # the bolted binary offsets) because BOLT may change where it puts instructions diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp index bc9cc8ce6cf5..86a287db72a4 100644 --- a/clang/lib/Driver/ToolChains/Clang.cpp +++ b/clang/lib/Driver/ToolChains/Clang.cpp @@ -5863,8 +5863,8 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA, } else if (Triple.getArch() == llvm::Triple::x86_64) { Ok = llvm::is_contained({"small", "kernel", "medium", "large", "tiny"}, CM); - } else if (Triple.isNVPTX() || Triple.isAMDGPU()) { - // NVPTX/AMDGPU does not care about the code model and will accept + } else if (Triple.isNVPTX() || Triple.isAMDGPU() || Triple.isSPIRV()) { + // NVPTX/AMDGPU/SPIRV does not care about the code model and will accept // whatever works for the host. Ok = true; } else if (Triple.isSPARC64()) { diff --git a/clang/test/Driver/unsupported-option-gpu.c b/clang/test/Driver/unsupported-option-gpu.c index f23cb71ebfb0..5618b2cba72e 100644 --- a/clang/test/Driver/unsupported-option-gpu.c +++ b/clang/test/Driver/unsupported-option-gpu.c @@ -2,4 +2,5 @@ // DEFINE: %{check} = %clang -### --target=x86_64-linux-gnu -c -mcmodel=medium // RUN: %{check} -x cuda %s --cuda-path=%S/Inputs/CUDA/usr/local/cuda --offload-arch=sm_60 --no-cuda-version-check -fbasic-block-sections=all +// RUN: %{check} -x hip %s --offload=spirv64 -nogpulib -nogpuinc // RUN: %{check} -x hip %s --rocm-path=%S/Inputs/rocm -nogpulib -nogpuinc diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp index 36248925d65a..919a14b8bcf0 100644 --- a/lld/MachO/Driver.cpp +++ b/lld/MachO/Driver.cpp @@ -612,7 +612,7 @@ static void replaceCommonSymbols() { if (!osec) osec = ConcatOutputSection::getOrCreateForInput(isec); isec->parent = osec; - inputSections.push_back(isec); + addInputSection(isec); // FIXME: CommonSymbol should store isReferencedDynamically, noDeadStrip // and pass them on here. @@ -1220,53 +1220,18 @@ static void createFiles(const InputArgList &args) { static void gatherInputSections() { TimeTraceScope timeScope("Gathering input sections"); - int inputOrder = 0; for (const InputFile *file : inputFiles) { for (const Section *section : file->sections) { // Compact unwind entries require special handling elsewhere. (In // contrast, EH frames are handled like regular ConcatInputSections.) if (section->name == section_names::compactUnwind) continue; - ConcatOutputSection *osec = nullptr; - for (const Subsection &subsection : section->subsections) { - if (auto *isec = dyn_cast(subsection.isec)) { - if (isec->isCoalescedWeak()) - continue; - if (config->emitInitOffsets && - sectionType(isec->getFlags()) == S_MOD_INIT_FUNC_POINTERS) { - in.initOffsets->addInput(isec); - continue; - } - isec->outSecOff = inputOrder++; - if (!osec) - osec = ConcatOutputSection::getOrCreateForInput(isec); - isec->parent = osec; - inputSections.push_back(isec); - } else if (auto *isec = - dyn_cast(subsection.isec)) { - if (isec->getName() == section_names::objcMethname) { - if (in.objcMethnameSection->inputOrder == UnspecifiedInputOrder) - in.objcMethnameSection->inputOrder = inputOrder++; - in.objcMethnameSection->addInput(isec); - } else { - if (in.cStringSection->inputOrder == UnspecifiedInputOrder) - in.cStringSection->inputOrder = inputOrder++; - in.cStringSection->addInput(isec); - } - } else if (auto *isec = - dyn_cast(subsection.isec)) { - if (in.wordLiteralSection->inputOrder == UnspecifiedInputOrder) - in.wordLiteralSection->inputOrder = inputOrder++; - in.wordLiteralSection->addInput(isec); - } else { - llvm_unreachable("unexpected input section kind"); - } - } + for (const Subsection &subsection : section->subsections) + addInputSection(subsection.isec); } if (!file->objCImageInfo.empty()) in.objCImageInfo->addFile(file); } - assert(inputOrder <= UnspecifiedInputOrder); } static void foldIdenticalLiterals() { @@ -1422,6 +1387,7 @@ bool link(ArrayRef argsArr, llvm::raw_ostream &stdoutOS, concatOutputSections.clear(); inputFiles.clear(); inputSections.clear(); + inputSectionsOrder = 0; loadedArchives.clear(); loadedObjectFrameworks.clear(); missingAutolinkWarnings.clear(); diff --git a/lld/MachO/InputSection.cpp b/lld/MachO/InputSection.cpp index 8f5affb1dc21..22930d52dd1d 100644 --- a/lld/MachO/InputSection.cpp +++ b/lld/MachO/InputSection.cpp @@ -37,6 +37,44 @@ static_assert(sizeof(void *) != 8 || "instances of it"); std::vector macho::inputSections; +int macho::inputSectionsOrder = 0; + +// Call this function to add a new InputSection and have it routed to the +// appropriate container. Depending on its type and current config, it will +// either be added to 'inputSections' vector or to a synthetic section. +void lld::macho::addInputSection(InputSection *inputSection) { + if (auto *isec = dyn_cast(inputSection)) { + if (isec->isCoalescedWeak()) + return; + if (config->emitInitOffsets && + sectionType(isec->getFlags()) == S_MOD_INIT_FUNC_POINTERS) { + in.initOffsets->addInput(isec); + return; + } + isec->outSecOff = inputSectionsOrder++; + auto *osec = ConcatOutputSection::getOrCreateForInput(isec); + isec->parent = osec; + inputSections.push_back(isec); + } else if (auto *isec = dyn_cast(inputSection)) { + if (isec->getName() == section_names::objcMethname) { + if (in.objcMethnameSection->inputOrder == UnspecifiedInputOrder) + in.objcMethnameSection->inputOrder = inputSectionsOrder++; + in.objcMethnameSection->addInput(isec); + } else { + if (in.cStringSection->inputOrder == UnspecifiedInputOrder) + in.cStringSection->inputOrder = inputSectionsOrder++; + in.cStringSection->addInput(isec); + } + } else if (auto *isec = dyn_cast(inputSection)) { + if (in.wordLiteralSection->inputOrder == UnspecifiedInputOrder) + in.wordLiteralSection->inputOrder = inputSectionsOrder++; + in.wordLiteralSection->addInput(isec); + } else { + llvm_unreachable("unexpected input section kind"); + } + + assert(inputSectionsOrder <= UnspecifiedInputOrder); +} uint64_t InputSection::getFileSize() const { return isZeroFill(getFlags()) ? 0 : getSize(); diff --git a/lld/MachO/InputSection.h b/lld/MachO/InputSection.h index b25f0638f4c6..694bdf734907 100644 --- a/lld/MachO/InputSection.h +++ b/lld/MachO/InputSection.h @@ -302,6 +302,8 @@ bool isEhFrameSection(const InputSection *); bool isGccExceptTabSection(const InputSection *); extern std::vector inputSections; +// This is used as a counter for specyfing input order for input sections +extern int inputSectionsOrder; namespace section_names { @@ -369,6 +371,7 @@ constexpr const char addrSig[] = "__llvm_addrsig"; } // namespace section_names +void addInputSection(InputSection *inputSection); } // namespace macho std::string toString(const macho::InputSection *); diff --git a/lld/MachO/ObjC.cpp b/lld/MachO/ObjC.cpp index 40df2243b26f..5902b82d30f5 100644 --- a/lld/MachO/ObjC.cpp +++ b/lld/MachO/ObjC.cpp @@ -790,7 +790,7 @@ void ObjcCategoryMerger::emitAndLinkProtocolList( infoCategoryWriter.catPtrListInfo.align); listSec->parent = infoCategoryWriter.catPtrListInfo.outputSection; listSec->live = true; - allInputSections.push_back(listSec); + addInputSection(listSec); listSec->parent = infoCategoryWriter.catPtrListInfo.outputSection; @@ -848,7 +848,7 @@ void ObjcCategoryMerger::emitAndLinkPointerList( infoCategoryWriter.catPtrListInfo.align); listSec->parent = infoCategoryWriter.catPtrListInfo.outputSection; listSec->live = true; - allInputSections.push_back(listSec); + addInputSection(listSec); listSec->parent = infoCategoryWriter.catPtrListInfo.outputSection; @@ -889,7 +889,7 @@ ObjcCategoryMerger::emitCatListEntrySec(const std::string &forCateogryName, bodyData, infoCategoryWriter.catListInfo.align); newCatList->parent = infoCategoryWriter.catListInfo.outputSection; newCatList->live = true; - allInputSections.push_back(newCatList); + addInputSection(newCatList); newCatList->parent = infoCategoryWriter.catListInfo.outputSection; @@ -927,7 +927,7 @@ Defined *ObjcCategoryMerger::emitCategoryBody(const std::string &name, bodyData, infoCategoryWriter.catBodyInfo.align); newBodySec->parent = infoCategoryWriter.catBodyInfo.outputSection; newBodySec->live = true; - allInputSections.push_back(newBodySec); + addInputSection(newBodySec); std::string symName = objc::symbol_names::category + baseClassName + "_$_(" + name + ")"; @@ -1132,7 +1132,7 @@ void ObjcCategoryMerger::generateCatListForNonErasedCategories( infoCategoryWriter.catListInfo.align); listSec->parent = infoCategoryWriter.catListInfo.outputSection; listSec->live = true; - allInputSections.push_back(listSec); + addInputSection(listSec); std::string slotSymName = "<__objc_catlist slot for category "; slotSymName += nonErasedCatBody->getName(); @@ -1221,9 +1221,11 @@ void ObjcCategoryMerger::doCleanup() { generatedSectionData.clear(); } StringRef ObjcCategoryMerger::newStringData(const char *str) { uint32_t len = strlen(str); - auto &data = newSectionData(len + 1); + uint32_t bufSize = len + 1; + auto &data = newSectionData(bufSize); char *strData = reinterpret_cast(data.data()); - strncpy(strData, str, len); + // Copy the string chars and null-terminator + memcpy(strData, str, bufSize); return StringRef(strData, len); } diff --git a/lld/MachO/SyntheticSections.cpp b/lld/MachO/SyntheticSections.cpp index 7ee3261ce307..1b3694528de1 100644 --- a/lld/MachO/SyntheticSections.cpp +++ b/lld/MachO/SyntheticSections.cpp @@ -793,7 +793,7 @@ void StubHelperSection::setUp() { in.imageLoaderCache->parent = ConcatOutputSection::getOrCreateForInput(in.imageLoaderCache); - inputSections.push_back(in.imageLoaderCache); + addInputSection(in.imageLoaderCache); // Since this isn't in the symbol table or in any input file, the noDeadStrip // argument doesn't matter. dyldPrivate = @@ -855,7 +855,7 @@ ConcatInputSection *ObjCSelRefsSection::makeSelRef(StringRef methname) { /*addend=*/static_cast(methnameOffset), /*referent=*/in.objcMethnameSection->isec}); objcSelref->parent = ConcatOutputSection::getOrCreateForInput(objcSelref); - inputSections.push_back(objcSelref); + addInputSection(objcSelref); objcSelref->isFinal = true; methnameToSelref[CachedHashStringRef(methname)] = objcSelref; return objcSelref; -- GitLab From b0e23639c5b19030bee2b307173802914f64aad6 Mon Sep 17 00:00:00 2001 From: Amir Ayupov Date: Tue, 20 Feb 2024 16:35:08 -0800 Subject: [PATCH 068/306] [BOLT] Add BB index to BAT Add input basic block index to BAT metadata. This addresses the case where some basic blocks are eliminated, and output index is not equal to the input block index. These indices are used in non-stale-matching mode. Increases BAT section size to: - large binary: 39521512 bytes (1.02x original), - medium binary: 3799988 bytes (0.64x), - small binary: 920 bytes (0.64x). Test Plan: Updated bolt-address-translation{,-yaml}.test Pull Request: https://github.com/llvm/llvm-project/pull/86044 --- bolt/docs/BAT.md | 11 +++--- .../bolt/Profile/BoltAddressTranslation.h | 7 +++- bolt/lib/Profile/BoltAddressTranslation.cpp | 39 ++++++++++++++----- .../X86/bolt-address-translation-yaml.test | 2 +- bolt/test/X86/bolt-address-translation.test | 2 +- 5 files changed, 43 insertions(+), 18 deletions(-) diff --git a/bolt/docs/BAT.md b/bolt/docs/BAT.md index 186b0e5ea89d..436593478a39 100644 --- a/bolt/docs/BAT.md +++ b/bolt/docs/BAT.md @@ -90,11 +90,12 @@ current function. ### Address translation table Delta encoding means that only the difference with the previous corresponding entry is encoded. Input offsets implicitly start at zero. -| Entry | Encoding | Description | -| ------ | ------| ----------- | -| `OutputOffset` | Continuous, Delta, ULEB128 | Function offset in output binary | -| `InputOffset` | Optional, Delta, SLEB128 | Function offset in input binary with `BRANCHENTRY` LSB bit | -| `BBHash` | Optional, 8b | Basic block entries only: basic block hash in input binary | +| Entry | Encoding | Description | Branch/BB | +| ------ | ------| ----------- | ------ | +| `OutputOffset` | Continuous, Delta, ULEB128 | Function offset in output binary | Both | +| `InputOffset` | Optional, Delta, SLEB128 | Function offset in input binary with `BRANCHENTRY` LSB bit | Both | +| `BBHash` | Optional, 8b | Basic block hash in input binary | BB | +| `BBIdx` | Optional, Delta, ULEB128 | Basic block index in input binary | BB | `BRANCHENTRY` bit denotes whether a given offset pair is a control flow source (branch or call instruction). If not set, it signifies a control flow target diff --git a/bolt/include/bolt/Profile/BoltAddressTranslation.h b/bolt/include/bolt/Profile/BoltAddressTranslation.h index 1f53f6d344ad..eda2b318f0d0 100644 --- a/bolt/include/bolt/Profile/BoltAddressTranslation.h +++ b/bolt/include/bolt/Profile/BoltAddressTranslation.h @@ -122,6 +122,10 @@ public: /// Returns BF hash by function output address (after BOLT). size_t getBFHash(uint64_t OutputAddress) const; + /// Returns BB index by function output address (after BOLT) and basic block + /// input offset. + unsigned getBBIndex(uint64_t FuncOutputAddress, uint32_t BBInputOffset) const; + /// True if a given \p Address is a function with translation table entry. bool isBATFunction(uint64_t Address) const { return Maps.count(Address); } @@ -154,7 +158,8 @@ private: std::map Maps; - using BBHashMap = std::unordered_map; + /// Map basic block input offset to a basic block index and hash pair. + using BBHashMap = std::unordered_map>; std::unordered_map> FuncHashes; /// Links outlined cold bocks to their original function diff --git a/bolt/lib/Profile/BoltAddressTranslation.cpp b/bolt/lib/Profile/BoltAddressTranslation.cpp index 1d61a1b735b4..8fe976cc00e5 100644 --- a/bolt/lib/Profile/BoltAddressTranslation.cpp +++ b/bolt/lib/Profile/BoltAddressTranslation.cpp @@ -45,6 +45,8 @@ void BoltAddressTranslation::writeEntriesForBB(MapTy &Map, LLVM_DEBUG(dbgs() << formatv(" Hash: {0:x}\n", getBBHash(HotFuncAddress, BBInputOffset))); (void)HotFuncAddress; + LLVM_DEBUG(dbgs() << formatv(" Index: {0}\n", + getBBIndex(HotFuncAddress, BBInputOffset))); // In case of conflicts (same Key mapping to different Vals), the last // update takes precedence. Of course it is not ideal to have conflicts and // those happen when we have an empty BB that either contained only @@ -217,6 +219,7 @@ void BoltAddressTranslation::writeMaps(std::map &Maps, } size_t Index = 0; uint64_t InOffset = 0; + size_t PrevBBIndex = 0; // Output and Input addresses and delta-encoded for (std::pair &KeyVal : Map) { const uint64_t OutputAddress = KeyVal.first + Address; @@ -226,11 +229,15 @@ void BoltAddressTranslation::writeMaps(std::map &Maps, encodeSLEB128(KeyVal.second - InOffset, OS); InOffset = KeyVal.second; // Keeping InOffset as if BRANCHENTRY is encoded if ((InOffset & BRANCHENTRY) == 0) { - // Basic block hash - size_t BBHash = FuncHashPair.second[InOffset >> 1]; + unsigned BBIndex; + size_t BBHash; + std::tie(BBIndex, BBHash) = FuncHashPair.second[InOffset >> 1]; OS.write(reinterpret_cast(&BBHash), 8); - LLVM_DEBUG(dbgs() << formatv("{0:x} -> {1:x} {2:x}\n", KeyVal.first, - InOffset >> 1, BBHash)); + // Basic block index in the input binary + encodeULEB128(BBIndex - PrevBBIndex, OS); + PrevBBIndex = BBIndex; + LLVM_DEBUG(dbgs() << formatv("{0:x} -> {1:x} {2:x} {3}\n", KeyVal.first, + InOffset >> 1, BBHash, BBIndex)); } } } @@ -316,6 +323,7 @@ void BoltAddressTranslation::parseMaps(std::vector &HotFuncs, LLVM_DEBUG(dbgs() << "Parsing " << NumEntries << " entries for 0x" << Twine::utohexstr(Address) << "\n"); uint64_t InputOffset = 0; + size_t BBIndex = 0; for (uint32_t J = 0; J < NumEntries; ++J) { const uint64_t OutputDelta = DE.getULEB128(&Offset, &Err); const uint64_t OutputAddress = PrevAddress + OutputDelta; @@ -330,19 +338,25 @@ void BoltAddressTranslation::parseMaps(std::vector &HotFuncs, } Map.insert(std::pair(OutputOffset, InputOffset)); size_t BBHash = 0; + size_t BBIndexDelta = 0; const bool IsBranchEntry = InputOffset & BRANCHENTRY; if (!IsBranchEntry) { BBHash = DE.getU64(&Offset, &Err); + BBIndexDelta = DE.getULEB128(&Offset, &Err); + BBIndex += BBIndexDelta; // Map basic block hash to hot fragment by input offset - FuncHashes[HotAddress].second.emplace(InputOffset >> 1, BBHash); + FuncHashes[HotAddress].second.emplace(InputOffset >> 1, + std::pair(BBIndex, BBHash)); } LLVM_DEBUG({ dbgs() << formatv( "{0:x} -> {1:x} ({2}/{3}b -> {4}/{5}b), {6:x}", OutputOffset, InputOffset, OutputDelta, getULEB128Size(OutputDelta), InputDelta, (J < EqualElems) ? 0 : getSLEB128Size(InputDelta), OutputAddress); - if (BBHash) - dbgs() << formatv(" {0:x}", BBHash); + if (!IsBranchEntry) { + dbgs() << formatv(" {0:x} {1}/{2}b", BBHash, BBIndex, + getULEB128Size(BBIndexDelta)); + } dbgs() << '\n'; }); } @@ -494,14 +508,19 @@ void BoltAddressTranslation::saveMetadata(BinaryContext &BC) { FuncHashes[BF.getAddress()].first = BF.computeHash(); BF.computeBlockHashes(); for (const BinaryBasicBlock &BB : BF) - FuncHashes[BF.getAddress()].second.emplace(BB.getInputOffset(), - BB.getHash()); + FuncHashes[BF.getAddress()].second.emplace( + BB.getInputOffset(), std::pair(BB.getIndex(), BB.getHash())); } } +unsigned BoltAddressTranslation::getBBIndex(uint64_t FuncOutputAddress, + uint32_t BBInputOffset) const { + return FuncHashes.at(FuncOutputAddress).second.at(BBInputOffset).first; +} + size_t BoltAddressTranslation::getBBHash(uint64_t FuncOutputAddress, uint32_t BBInputOffset) const { - return FuncHashes.at(FuncOutputAddress).second.at(BBInputOffset); + return FuncHashes.at(FuncOutputAddress).second.at(BBInputOffset).second; } size_t BoltAddressTranslation::getBFHash(uint64_t OutputAddress) const { diff --git a/bolt/test/X86/bolt-address-translation-yaml.test b/bolt/test/X86/bolt-address-translation-yaml.test index 25ff4e7fbfcc..4516a662697a 100644 --- a/bolt/test/X86/bolt-address-translation-yaml.test +++ b/bolt/test/X86/bolt-address-translation-yaml.test @@ -18,7 +18,7 @@ RUN: | FileCheck --check-prefix CHECK-BOLT-YAML %s WRITE-BAT-CHECK: BOLT-INFO: Wrote 5 BAT maps WRITE-BAT-CHECK: BOLT-INFO: Wrote 4 function and 22 basic block hashes -WRITE-BAT-CHECK: BOLT-INFO: BAT section size (bytes): 344 +WRITE-BAT-CHECK: BOLT-INFO: BAT section size (bytes): 376 READ-BAT-CHECK-NOT: BOLT-ERROR: unable to save profile in YAML format for input file processed by BOLT READ-BAT-CHECK: BOLT-INFO: Parsed 5 BAT entries diff --git a/bolt/test/X86/bolt-address-translation.test b/bolt/test/X86/bolt-address-translation.test index 4277b4e0d0fe..5c1db89e3c6b 100644 --- a/bolt/test/X86/bolt-address-translation.test +++ b/bolt/test/X86/bolt-address-translation.test @@ -37,7 +37,7 @@ # CHECK: BOLT: 3 out of 7 functions were overwritten. # CHECK: BOLT-INFO: Wrote 6 BAT maps # CHECK: BOLT-INFO: Wrote 3 function and 58 basic block hashes -# CHECK: BOLT-INFO: BAT section size (bytes): 816 +# CHECK: BOLT-INFO: BAT section size (bytes): 920 # # usqrt mappings (hot part). We match against any key (left side containing # the bolted binary offsets) because BOLT may change where it puts instructions -- GitLab From cb300c33059c1d14f72392ce5dffcf050ad7567d Mon Sep 17 00:00:00 2001 From: Christian Ulmann Date: Fri, 22 Mar 2024 16:44:06 +0100 Subject: [PATCH 069/306] [MLIR][LLVM][SROA] Fix pointer escape through stores bug (#86291) This commit resolves a SROA bug caused by not properly checking if a llvm store operation writes the pointer to memory or not. Now, we do no longer consider stores that use a slot pointer as a value to store as fixable. --- mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp | 4 ++++ mlir/test/Dialect/LLVMIR/sroa.mlir | 13 +++++++++++++ 2 files changed, 17 insertions(+) diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp index 0ef1d105aca6..f171bf7cc4be 100644 --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp @@ -251,6 +251,10 @@ bool LLVM::StoreOp::canRewire(const DestructurableMemorySlot &slot, if (getVolatile_()) return false; + // Storing the pointer to memory cannot be dealt with. + if (getValue() == slot.ptr) + return false; + // A store always accesses the first element of the destructured slot. auto index = IntegerAttr::get(IntegerType::get(getContext(), 32), 0); Type subslotType = getTypeAtIndex(slot, index); diff --git a/mlir/test/Dialect/LLVMIR/sroa.mlir b/mlir/test/Dialect/LLVMIR/sroa.mlir index ca49b1298b0e..3f4d17c6a43f 100644 --- a/mlir/test/Dialect/LLVMIR/sroa.mlir +++ b/mlir/test/Dialect/LLVMIR/sroa.mlir @@ -305,3 +305,16 @@ llvm.func @vector_store_type_mismatch(%arg: vector<4xi32>) { llvm.store %arg, %1 : vector<4xi32>, !llvm.ptr llvm.return } + +// ----- + +// CHECK-LABEL: llvm.func @store_to_memory +// CHECK-SAME: %[[ARG:.*]]: !llvm.ptr +llvm.func @store_to_memory(%arg: !llvm.ptr) { + %0 = llvm.mlir.constant(1 : i32) : i32 + // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct< + %1 = llvm.alloca %0 x !llvm.struct<"foo", (vector<4xf32>)> : (i32) -> !llvm.ptr + // CHECK-NEXT: llvm.store %[[ALLOCA]], %[[ARG]] + llvm.store %1, %arg : !llvm.ptr, !llvm.ptr + llvm.return +} -- GitLab From ceba3a38e8f7b378ad20641832d568460892af1d Mon Sep 17 00:00:00 2001 From: Amir Ayupov Date: Fri, 22 Mar 2024 08:46:48 -0700 Subject: [PATCH 070/306] [BOLT] Add number of basic blocks to BAT YAML profile reader checks the number of basic blocks in regular, no-stale-matching mode. Add it to BAT. This increases the size of BAT section to: - large binary: 39583080 bytes (1.02x of the original), - medium binary: 3816492 bytes (0.64x), - small binary: 920 bytes (0.64x, no change due to alignment). Test Plan: Updated bolt-address-translation-yaml.test Reviewers: rafaelauler, ayermolo, maksfb, dcci Reviewed By: rafaelauler Pull Request: https://github.com/llvm/llvm-project/pull/86045 --- bolt/docs/BAT.md | 1 + bolt/include/bolt/Profile/BoltAddressTranslation.h | 3 +++ bolt/lib/Profile/BoltAddressTranslation.cpp | 10 ++++++++++ bolt/test/X86/bolt-address-translation-yaml.test | 2 +- 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/bolt/docs/BAT.md b/bolt/docs/BAT.md index 436593478a39..2279a070263e 100644 --- a/bolt/docs/BAT.md +++ b/bolt/docs/BAT.md @@ -79,6 +79,7 @@ Hot indices are delta encoded, implicitly starting at zero. | `Address` | Continuous, Delta, ULEB128 | Function address in the output binary | | `HotIndex` | Delta, ULEB128 | Cold functions only: index of corresponding hot function in hot functions table | | `FuncHash` | 8b | Hot functions only: function hash for input function | +| `NumBlocks` | ULEB128 | Hot functions only: number of basic blocks in the original function | | `NumEntries` | ULEB128 | Number of address translation entries for a function | | `EqualElems` | ULEB128 | Hot functions only: number of equal offsets in the beginning of a function | | `BranchEntries` | Bitmask, `alignTo(EqualElems, 8)` bits | Hot functions only: if `EqualElems` is non-zero, bitmask denoting entries with `BRANCHENTRY` bit | diff --git a/bolt/include/bolt/Profile/BoltAddressTranslation.h b/bolt/include/bolt/Profile/BoltAddressTranslation.h index eda2b318f0d0..d583ce0b76a2 100644 --- a/bolt/include/bolt/Profile/BoltAddressTranslation.h +++ b/bolt/include/bolt/Profile/BoltAddressTranslation.h @@ -162,6 +162,9 @@ private: using BBHashMap = std::unordered_map>; std::unordered_map> FuncHashes; + /// Map a function to its basic blocks count + std::unordered_map NumBasicBlocksMap; + /// Links outlined cold bocks to their original function std::map ColdPartSource; diff --git a/bolt/lib/Profile/BoltAddressTranslation.cpp b/bolt/lib/Profile/BoltAddressTranslation.cpp index 8fe976cc00e5..31886f4c8025 100644 --- a/bolt/lib/Profile/BoltAddressTranslation.cpp +++ b/bolt/lib/Profile/BoltAddressTranslation.cpp @@ -196,6 +196,10 @@ void BoltAddressTranslation::writeMaps(std::map &Maps, // Function hash LLVM_DEBUG(dbgs() << "Hash: " << formatv("{0:x}\n", FuncHashPair.first)); OS.write(reinterpret_cast(&FuncHashPair.first), 8); + // Number of basic blocks + size_t NumBasicBlocks = FuncHashPair.second.size(); + LLVM_DEBUG(dbgs() << "Basic blocks: " << NumBasicBlocks << '\n'); + encodeULEB128(NumBasicBlocks, OS); } encodeULEB128(NumEntries, OS); // For hot fragments only: encode the number of equal offsets @@ -293,6 +297,12 @@ void BoltAddressTranslation::parseMaps(std::vector &HotFuncs, const size_t FuncHash = DE.getU64(&Offset, &Err); FuncHashes[Address].first = FuncHash; LLVM_DEBUG(dbgs() << formatv("{0:x}: hash {1:x}\n", Address, FuncHash)); + // Number of basic blocks + const size_t NumBasicBlocks = DE.getULEB128(&Offset, &Err); + NumBasicBlocksMap.emplace(Address, NumBasicBlocks); + LLVM_DEBUG(dbgs() << formatv("{0:x}: #bbs {1}, {2} bytes\n", Address, + NumBasicBlocks, + getULEB128Size(NumBasicBlocks))); } const uint32_t NumEntries = DE.getULEB128(&Offset, &Err); // Equal offsets, hot fragments only. diff --git a/bolt/test/X86/bolt-address-translation-yaml.test b/bolt/test/X86/bolt-address-translation-yaml.test index 4516a662697a..ee54a90a9f2a 100644 --- a/bolt/test/X86/bolt-address-translation-yaml.test +++ b/bolt/test/X86/bolt-address-translation-yaml.test @@ -18,7 +18,7 @@ RUN: | FileCheck --check-prefix CHECK-BOLT-YAML %s WRITE-BAT-CHECK: BOLT-INFO: Wrote 5 BAT maps WRITE-BAT-CHECK: BOLT-INFO: Wrote 4 function and 22 basic block hashes -WRITE-BAT-CHECK: BOLT-INFO: BAT section size (bytes): 376 +WRITE-BAT-CHECK: BOLT-INFO: BAT section size (bytes): 380 READ-BAT-CHECK-NOT: BOLT-ERROR: unable to save profile in YAML format for input file processed by BOLT READ-BAT-CHECK: BOLT-INFO: Parsed 5 BAT entries -- GitLab From 2091c74796b1dac68e622284c63a870b88b7554f Mon Sep 17 00:00:00 2001 From: Orlando Cazalet-Hyams Date: Fri, 22 Mar 2024 15:47:40 +0000 Subject: [PATCH 071/306] [RemoveDIs] Update DIBuilder C API with DbgRecord functions [2/2] (#85657) Follow on from #84915 which adds the DbgRecord function variants. Update the LLVMDIBuilderInsert... functions to insert DbgRecords instead of debug intrinsics. LLVMDIBuilderInsertDeclareBefore LLVMDIBuilderInsertDeclareAtEnd LLVMDIBuilderInsertDbgValueBefore LLVMDIBuilderInsertDbgValueAtEnd Calling these functions will now cause an assertion if the module is in the wrong debug info format. They should only be used when the module is in "new debug format". Use LLVMIsNewDbgInfoFormat to query and LLVMSetIsNewDbgInfoFormat to change the debug info format of a module. Please see https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-change (RemoveDIsDebugInfo.md) for more info. --- llvm/docs/RemoveDIsDebugInfo.md | 11 ++- llvm/include/llvm-c/DebugInfo.h | 60 +++++++++----- llvm/lib/IR/DebugInfo.cpp | 121 +++++++++++++++++++---------- llvm/tools/llvm-c-test/debuginfo.c | 13 ++-- 4 files changed, 137 insertions(+), 68 deletions(-) diff --git a/llvm/docs/RemoveDIsDebugInfo.md b/llvm/docs/RemoveDIsDebugInfo.md index a2f1e173d9d9..9e50a2a604aa 100644 --- a/llvm/docs/RemoveDIsDebugInfo.md +++ b/llvm/docs/RemoveDIsDebugInfo.md @@ -40,15 +40,22 @@ New functions (all to be deprecated) LLVMIsNewDbgInfoFormat # Returns true if the module is in the new non-instruction mode. LLVMSetIsNewDbgInfoFormat # Convert to the requested debug info format. -LLVMDIBuilderInsertDeclareIntrinsicBefore # Insert a debug intrinsic (old debug info format). +LLVMDIBuilderInsertDeclareIntrinsicBefore # Insert a debug intrinsic (old debug info format). LLVMDIBuilderInsertDeclareIntrinsicAtEnd # Same as above. LLVMDIBuilderInsertDbgValueIntrinsicBefore # Same as above. LLVMDIBuilderInsertDbgValueIntrinsicAtEnd # Same as above. -LLVMDIBuilderInsertDeclareRecordBefore # Insert a debug record (new debug info format). +LLVMDIBuilderInsertDeclareRecordBefore # Insert a debug record (new debug info format). LLVMDIBuilderInsertDeclareRecordAtEnd # Same as above. LLVMDIBuilderInsertDbgValueRecordBefore # Same as above. LLVMDIBuilderInsertDbgValueRecordAtEnd # Same as above. + +Existing functions (behaviour change) +------------------------------------- +LLVMDIBuilderInsertDeclareBefore # Insert a debug record (new debug info format) instead of a debug intrinsic (old debug info format). +LLVMDIBuilderInsertDeclareAtEnd # Same as above. +LLVMDIBuilderInsertDbgValueBefore # Same as above. +LLVMDIBuilderInsertDbgValueAtEnd # Same as above. ``` # Anything else? diff --git a/llvm/include/llvm-c/DebugInfo.h b/llvm/include/llvm-c/DebugInfo.h index b23ff63c862f..dab1d697761b 100644 --- a/llvm/include/llvm-c/DebugInfo.h +++ b/llvm/include/llvm-c/DebugInfo.h @@ -1249,7 +1249,12 @@ LLVMMetadataRef LLVMDIBuilderCreateTempGlobalVariableFwdDecl( LLVMMetadataRef Decl, uint32_t AlignInBits); /* - * Insert a new llvm.dbg.declare intrinsic call before the given instruction. + * Insert a new Declare DbgRecord before the given instruction. + * + * Only use in "new debug mode" (LLVMIsNewDbgInfoFormat() is true). + * Use LLVMSetIsNewDbgInfoFormat(LLVMBool) to convert between formats. + * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes + * * \param Builder The DIBuilder. * \param Storage The storage of the variable to declare. * \param VarInfo The variable's debug info descriptor. @@ -1257,13 +1262,13 @@ LLVMMetadataRef LLVMDIBuilderCreateTempGlobalVariableFwdDecl( * \param DebugLoc Debug info location. * \param Instr Instruction acting as a location for the new intrinsic. */ -LLVMValueRef +LLVMDbgRecordRef LLVMDIBuilderInsertDeclareBefore(LLVMDIBuilderRef Builder, LLVMValueRef Storage, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMValueRef Instr); /** * Soon to be deprecated. - * Only use in "old debug mode" (LLVMIsNewDbgFormat() is false). + * Only use in "old debug mode" (LLVMIsNewDbgInfoFormat() is false). * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes * * Insert a new llvm.dbg.declare intrinsic call before the given instruction. @@ -1279,7 +1284,7 @@ LLVMValueRef LLVMDIBuilderInsertDeclareIntrinsicBefore( LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMValueRef Instr); /** * Soon to be deprecated. - * Only use in "new debug mode" (LLVMIsNewDbgFormat() is true). + * Only use in "new debug mode" (LLVMIsNewDbgInfoFormat() is true). * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes * * Insert a Declare DbgRecord before the given instruction. @@ -1295,9 +1300,14 @@ LLVMDbgRecordRef LLVMDIBuilderInsertDeclareRecordBefore( LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMValueRef Instr); /** - * Insert a new llvm.dbg.declare intrinsic call at the end of the given basic - * block. If the basic block has a terminator instruction, the intrinsic is - * inserted before that terminator instruction. + * Insert a new Declare DbgRecord at the end of the given basic block. If the + * basic block has a terminator instruction, the intrinsic is inserted before + * that terminator instruction. + * + * Only use in "new debug mode" (LLVMIsNewDbgInfoFormat() is true). + * Use LLVMSetIsNewDbgInfoFormat(LLVMBool) to convert between formats. + * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes + * * \param Builder The DIBuilder. * \param Storage The storage of the variable to declare. * \param VarInfo The variable's debug info descriptor. @@ -1305,12 +1315,12 @@ LLVMDbgRecordRef LLVMDIBuilderInsertDeclareRecordBefore( * \param DebugLoc Debug info location. * \param Block Basic block acting as a location for the new intrinsic. */ -LLVMValueRef LLVMDIBuilderInsertDeclareAtEnd( +LLVMDbgRecordRef LLVMDIBuilderInsertDeclareAtEnd( LLVMDIBuilderRef Builder, LLVMValueRef Storage, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMBasicBlockRef Block); /** * Soon to be deprecated. - * Only use in "old debug mode" (LLVMIsNewDbgFormat() is false). + * Only use in "old debug mode" (LLVMIsNewDbgInfoFormat() is false). * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes * * Insert a new llvm.dbg.declare intrinsic call at the end of the given basic @@ -1328,7 +1338,7 @@ LLVMValueRef LLVMDIBuilderInsertDeclareIntrinsicAtEnd( LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMBasicBlockRef Block); /** * Soon to be deprecated. - * Only use in "new debug mode" (LLVMIsNewDbgFormat() is true). + * Only use in "new debug mode" (LLVMIsNewDbgInfoFormat() is true). * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes * * Insert a Declare DbgRecord at the end of the given basic block. If the basic @@ -1346,7 +1356,12 @@ LLVMDbgRecordRef LLVMDIBuilderInsertDeclareRecordAtEnd( LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMBasicBlockRef Block); /** - * Insert a new llvm.dbg.value intrinsic call before the given instruction. + * Insert a new Value DbgRecord before the given instruction. + * + * Only use in "new debug mode" (LLVMIsNewDbgInfoFormat() is true). + * Use LLVMSetIsNewDbgInfoFormat(LLVMBool) to convert between formats. + * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes + * * \param Builder The DIBuilder. * \param Val The value of the variable. * \param VarInfo The variable's debug info descriptor. @@ -1354,13 +1369,13 @@ LLVMDbgRecordRef LLVMDIBuilderInsertDeclareRecordAtEnd( * \param DebugLoc Debug info location. * \param Instr Instruction acting as a location for the new intrinsic. */ -LLVMValueRef +LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueBefore(LLVMDIBuilderRef Builder, LLVMValueRef Val, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMValueRef Instr); /** * Soon to be deprecated. - * Only use in "old debug mode" (Module::IsNewDbgInfoFormat is false). + * Only use in "old debug mode" (LLVMIsNewDbgInfoFormat() is false). * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes * * Insert a new llvm.dbg.value intrinsic call before the given instruction. @@ -1376,7 +1391,7 @@ LLVMValueRef LLVMDIBuilderInsertDbgValueIntrinsicBefore( LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMValueRef Instr); /** * Soon to be deprecated. - * Only use in "new debug mode" (Module::IsNewDbgInfoFormat is true). + * Only use in "new debug mode" (LLVMIsNewDbgInfoFormat() is true). * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes * * Insert a new llvm.dbg.value intrinsic call before the given instruction. @@ -1392,9 +1407,14 @@ LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueRecordBefore( LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMValueRef Instr); /** - * Insert a new llvm.dbg.value intrinsic call at the end of the given basic - * block. If the basic block has a terminator instruction, the intrinsic is - * inserted before that terminator instruction. + * Insert a new Value DbgRecord at the end of the given basic block. If the + * basic block has a terminator instruction, the intrinsic is inserted before + * that terminator instruction. + * + * Only use in "new debug mode" (LLVMIsNewDbgInfoFormat() is true). + * Use LLVMSetIsNewDbgInfoFormat(LLVMBool) to convert between formats. + * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes + * * \param Builder The DIBuilder. * \param Val The value of the variable. * \param VarInfo The variable's debug info descriptor. @@ -1402,12 +1422,12 @@ LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueRecordBefore( * \param DebugLoc Debug info location. * \param Block Basic block acting as a location for the new intrinsic. */ -LLVMValueRef LLVMDIBuilderInsertDbgValueAtEnd( +LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueAtEnd( LLVMDIBuilderRef Builder, LLVMValueRef Val, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMBasicBlockRef Block); /** * Soon to be deprecated. - * Only use in "old debug mode" (Module::IsNewDbgInfoFormat is false). + * Only use in "old debug mode" (LLVMIsNewDbgInfoFormat() is false). * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes * * Insert a new llvm.dbg.value intrinsic call at the end of the given basic @@ -1425,7 +1445,7 @@ LLVMValueRef LLVMDIBuilderInsertDbgValueIntrinsicAtEnd( LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMBasicBlockRef Block); /** * Soon to be deprecated. - * Only use in "new debug mode" (Module::IsNewDbgInfoFormat is true). + * Only use in "new debug mode" (LLVMIsNewDbgInfoFormat() is true). * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes * * Insert a new llvm.dbg.value intrinsic call at the end of the given basic diff --git a/llvm/lib/IR/DebugInfo.cpp b/llvm/lib/IR/DebugInfo.cpp index 09bce9df1f33..4206162d1768 100644 --- a/llvm/lib/IR/DebugInfo.cpp +++ b/llvm/lib/IR/DebugInfo.cpp @@ -1665,12 +1665,12 @@ LLVMMetadataRef LLVMDIBuilderCreateTempGlobalVariableFwdDecl( unwrapDI(Decl), nullptr, AlignInBits)); } -LLVMValueRef +LLVMDbgRecordRef LLVMDIBuilderInsertDeclareBefore(LLVMDIBuilderRef Builder, LLVMValueRef Storage, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DL, LLVMValueRef Instr) { - return LLVMDIBuilderInsertDeclareIntrinsicBefore(Builder, Storage, VarInfo, - Expr, DL, Instr); + return LLVMDIBuilderInsertDeclareRecordBefore(Builder, Storage, VarInfo, Expr, + DL, Instr); } LLVMValueRef LLVMDIBuilderInsertDeclareIntrinsicBefore( LLVMDIBuilderRef Builder, LLVMValueRef Storage, LLVMMetadataRef VarInfo, @@ -1679,27 +1679,38 @@ LLVMValueRef LLVMDIBuilderInsertDeclareIntrinsicBefore( unwrap(Storage), unwrap(VarInfo), unwrap(Expr), unwrap(DL), unwrap(Instr)); + // This assert will fail if the module is in the new debug info format. + // This function should only be called if the module is in the old + // debug info format. + // See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes, + // LLVMIsNewDbgInfoFormat, and LLVMSetIsNewDbgInfoFormat for more info. assert(isa(DbgInst) && - "Inserted a DbgRecord into function using old debug info mode"); + "Function unexpectedly in new debug info format"); return wrap(cast(DbgInst)); } LLVMDbgRecordRef LLVMDIBuilderInsertDeclareRecordBefore( LLVMDIBuilderRef Builder, LLVMValueRef Storage, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DL, LLVMValueRef Instr) { - return wrap( - unwrap(Builder) - ->insertDeclare(unwrap(Storage), unwrap(VarInfo), - unwrap(Expr), unwrap(DL), - unwrap(Instr)) - .get()); + DbgInstPtr DbgInst = unwrap(Builder)->insertDeclare( + unwrap(Storage), unwrap(VarInfo), + unwrap(Expr), unwrap(DL), + unwrap(Instr)); + // This assert will fail if the module is in the old debug info format. + // This function should only be called if the module is in the new + // debug info format. + // See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes, + // LLVMIsNewDbgInfoFormat, and LLVMSetIsNewDbgInfoFormat for more info. + assert(isa(DbgInst) && + "Function unexpectedly in old debug info format"); + return wrap(cast(DbgInst)); } -LLVMValueRef +LLVMDbgRecordRef LLVMDIBuilderInsertDeclareAtEnd(LLVMDIBuilderRef Builder, LLVMValueRef Storage, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DL, LLVMBasicBlockRef Block) { - return LLVMDIBuilderInsertDeclareIntrinsicAtEnd(Builder, Storage, VarInfo, - Expr, DL, Block); + return LLVMDIBuilderInsertDeclareRecordAtEnd(Builder, Storage, VarInfo, Expr, + DL, Block); } LLVMValueRef LLVMDIBuilderInsertDeclareIntrinsicAtEnd( LLVMDIBuilderRef Builder, LLVMValueRef Storage, LLVMMetadataRef VarInfo, @@ -1707,26 +1718,36 @@ LLVMValueRef LLVMDIBuilderInsertDeclareIntrinsicAtEnd( DbgInstPtr DbgInst = unwrap(Builder)->insertDeclare( unwrap(Storage), unwrap(VarInfo), unwrap(Expr), unwrap(DL), unwrap(Block)); + // This assert will fail if the module is in the new debug info format. + // This function should only be called if the module is in the old + // debug info format. + // See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes, + // LLVMIsNewDbgInfoFormat, and LLVMSetIsNewDbgInfoFormat for more info. assert(isa(DbgInst) && - "Inserted a DbgRecord into function using old debug info mode"); + "Function unexpectedly in new debug info format"); return wrap(cast(DbgInst)); } LLVMDbgRecordRef LLVMDIBuilderInsertDeclareRecordAtEnd( LLVMDIBuilderRef Builder, LLVMValueRef Storage, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DL, LLVMBasicBlockRef Block) { - return wrap(unwrap(Builder) - ->insertDeclare(unwrap(Storage), - unwrap(VarInfo), - unwrap(Expr), - unwrap(DL), unwrap(Block)) - .get()); + DbgInstPtr DbgInst = unwrap(Builder)->insertDeclare( + unwrap(Storage), unwrap(VarInfo), + unwrap(Expr), unwrap(DL), unwrap(Block)); + // This assert will fail if the module is in the old debug info format. + // This function should only be called if the module is in the new + // debug info format. + // See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes, + // LLVMIsNewDbgInfoFormat, and LLVMSetIsNewDbgInfoFormat for more info. + assert(isa(DbgInst) && + "Function unexpectedly in old debug info format"); + return wrap(cast(DbgInst)); } -LLVMValueRef LLVMDIBuilderInsertDbgValueBefore( +LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueBefore( LLVMDIBuilderRef Builder, LLVMValueRef Val, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMValueRef Instr) { - return LLVMDIBuilderInsertDbgValueIntrinsicBefore(Builder, Val, VarInfo, Expr, - DebugLoc, Instr); + return LLVMDIBuilderInsertDbgValueRecordBefore(Builder, Val, VarInfo, Expr, + DebugLoc, Instr); } LLVMValueRef LLVMDIBuilderInsertDbgValueIntrinsicBefore( LLVMDIBuilderRef Builder, LLVMValueRef Val, LLVMMetadataRef VarInfo, @@ -1734,26 +1755,36 @@ LLVMValueRef LLVMDIBuilderInsertDbgValueIntrinsicBefore( DbgInstPtr DbgInst = unwrap(Builder)->insertDbgValueIntrinsic( unwrap(Val), unwrap(VarInfo), unwrap(Expr), unwrap(DebugLoc), unwrap(Instr)); + // This assert will fail if the module is in the new debug info format. + // This function should only be called if the module is in the old + // debug info format. + // See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes, + // LLVMIsNewDbgInfoFormat, and LLVMSetIsNewDbgInfoFormat for more info. assert(isa(DbgInst) && - "Inserted a DbgRecord into function using old debug info mode"); + "Function unexpectedly in new debug info format"); return wrap(cast(DbgInst)); } LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueRecordBefore( LLVMDIBuilderRef Builder, LLVMValueRef Val, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMValueRef Instr) { - return wrap(unwrap(Builder) - ->insertDbgValueIntrinsic( - unwrap(Val), unwrap(VarInfo), - unwrap(Expr), unwrap(DebugLoc), - unwrap(Instr)) - .get()); + DbgInstPtr DbgInst = unwrap(Builder)->insertDbgValueIntrinsic( + unwrap(Val), unwrap(VarInfo), unwrap(Expr), + unwrap(DebugLoc), unwrap(Instr)); + // This assert will fail if the module is in the old debug info format. + // This function should only be called if the module is in the new + // debug info format. + // See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes, + // LLVMIsNewDbgInfoFormat, and LLVMSetIsNewDbgInfoFormat for more info. + assert(isa(DbgInst) && + "Function unexpectedly in old debug info format"); + return wrap(cast(DbgInst)); } -LLVMValueRef LLVMDIBuilderInsertDbgValueAtEnd( +LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueAtEnd( LLVMDIBuilderRef Builder, LLVMValueRef Val, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMBasicBlockRef Block) { - return LLVMDIBuilderInsertDbgValueIntrinsicAtEnd(Builder, Val, VarInfo, Expr, - DebugLoc, Block); + return LLVMDIBuilderInsertDbgValueRecordAtEnd(Builder, Val, VarInfo, Expr, + DebugLoc, Block); } LLVMValueRef LLVMDIBuilderInsertDbgValueIntrinsicAtEnd( LLVMDIBuilderRef Builder, LLVMValueRef Val, LLVMMetadataRef VarInfo, @@ -1761,19 +1792,29 @@ LLVMValueRef LLVMDIBuilderInsertDbgValueIntrinsicAtEnd( DbgInstPtr DbgInst = unwrap(Builder)->insertDbgValueIntrinsic( unwrap(Val), unwrap(VarInfo), unwrap(Expr), unwrap(DebugLoc), unwrap(Block)); + // This assert will fail if the module is in the new debug info format. + // This function should only be called if the module is in the old + // debug info format. + // See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes, + // LLVMIsNewDbgInfoFormat, and LLVMSetIsNewDbgInfoFormat for more info. assert(isa(DbgInst) && - "Inserted a DbgRecord into function using old debug info mode"); + "Function unexpectedly in new debug info format"); return wrap(cast(DbgInst)); } LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueRecordAtEnd( LLVMDIBuilderRef Builder, LLVMValueRef Val, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMBasicBlockRef Block) { - return wrap(unwrap(Builder) - ->insertDbgValueIntrinsic( - unwrap(Val), unwrap(VarInfo), - unwrap(Expr), unwrap(DebugLoc), - unwrap(Block)) - .get()); + DbgInstPtr DbgInst = unwrap(Builder)->insertDbgValueIntrinsic( + unwrap(Val), unwrap(VarInfo), unwrap(Expr), + unwrap(DebugLoc), unwrap(Block)); + // This assert will fail if the module is in the old debug info format. + // This function should only be called if the module is in the new + // debug info format. + // See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes, + // LLVMIsNewDbgInfoFormat, and LLVMSetIsNewDbgInfoFormat for more info. + assert(isa(DbgInst) && + "Function unexpectedly in old debug info format"); + return wrap(cast(DbgInst)); } LLVMMetadataRef LLVMDIBuilderCreateAutoVariable( diff --git a/llvm/tools/llvm-c-test/debuginfo.c b/llvm/tools/llvm-c-test/debuginfo.c index 78ccaf12a380..9b5c37b05d90 100644 --- a/llvm/tools/llvm-c-test/debuginfo.c +++ b/llvm/tools/llvm-c-test/debuginfo.c @@ -136,12 +136,13 @@ int llvm_test_dibuilder(bool NewDebugInfoFormat) { LLVMMetadataRef FooParamVar1 = LLVMDIBuilderCreateParameterVariable(DIB, FunctionMetadata, "a", 1, 1, File, 42, Int64Ty, true, 0); + if (LLVMIsNewDbgInfoFormat(M)) - LLVMDIBuilderInsertDeclareRecordAtEnd( + LLVMDIBuilderInsertDeclareAtEnd( DIB, LLVMConstInt(LLVMInt64Type(), 0, false), FooParamVar1, FooParamExpression, FooParamLocation, FooEntryBlock); else - LLVMDIBuilderInsertDeclareAtEnd( + LLVMDIBuilderInsertDeclareIntrinsicAtEnd( DIB, LLVMConstInt(LLVMInt64Type(), 0, false), FooParamVar1, FooParamExpression, FooParamLocation, FooEntryBlock); LLVMMetadataRef FooParamVar2 = @@ -149,11 +150,11 @@ int llvm_test_dibuilder(bool NewDebugInfoFormat) { 42, Int64Ty, true, 0); if (LLVMIsNewDbgInfoFormat(M)) - LLVMDIBuilderInsertDeclareRecordAtEnd( + LLVMDIBuilderInsertDeclareAtEnd( DIB, LLVMConstInt(LLVMInt64Type(), 0, false), FooParamVar2, FooParamExpression, FooParamLocation, FooEntryBlock); else - LLVMDIBuilderInsertDeclareAtEnd( + LLVMDIBuilderInsertDeclareIntrinsicAtEnd( DIB, LLVMConstInt(LLVMInt64Type(), 0, false), FooParamVar2, FooParamExpression, FooParamLocation, FooEntryBlock); @@ -161,11 +162,11 @@ int llvm_test_dibuilder(bool NewDebugInfoFormat) { LLVMDIBuilderCreateParameterVariable(DIB, FunctionMetadata, "c", 1, 3, File, 42, VectorTy, true, 0); if (LLVMIsNewDbgInfoFormat(M)) - LLVMDIBuilderInsertDeclareRecordAtEnd( + LLVMDIBuilderInsertDeclareAtEnd( DIB, LLVMConstInt(LLVMInt64Type(), 0, false), FooParamVar3, FooParamExpression, FooParamLocation, FooEntryBlock); else - LLVMDIBuilderInsertDeclareAtEnd( + LLVMDIBuilderInsertDeclareIntrinsicAtEnd( DIB, LLVMConstInt(LLVMInt64Type(), 0, false), FooParamVar3, FooParamExpression, FooParamLocation, FooEntryBlock); -- GitLab From 3054d0dae7a813c493d2bb8e969aa2321145a83b Mon Sep 17 00:00:00 2001 From: Changpeng Fang Date: Fri, 22 Mar 2024 08:51:53 -0700 Subject: [PATCH 072/306] AMDGPU: Rename and add bf16 support for global_load_tr builtins (#86202) Make the name of a clang builtin as close to the mnemonic instruction name as possible. The data type suffix may not be enough to tell what instruction the builtin is going to produce. This patch also add the bf16 support for global_load_tr_b128 builtins. --- clang/include/clang/Basic/BuiltinsAMDGPU.def | 16 ++++---- clang/lib/CodeGen/CGBuiltin.cpp | 34 +++++++++++------ ...uiltins-amdgcn-global-load-tr-gfx11-err.cl | 25 ++++++------ ...ins-amdgcn-global-load-tr-gfx12-w32-err.cl | 11 +++--- ...ins-amdgcn-global-load-tr-gfx12-w64-err.cl | 11 +++--- .../builtins-amdgcn-global-load-tr-w32.cl | 38 +++++++++---------- .../builtins-amdgcn-global-load-tr-w64.cl | 38 +++++++++---------- 7 files changed, 94 insertions(+), 79 deletions(-) diff --git a/clang/include/clang/Basic/BuiltinsAMDGPU.def b/clang/include/clang/Basic/BuiltinsAMDGPU.def index 61ec8b79bf05..4153b316c22b 100644 --- a/clang/include/clang/Basic/BuiltinsAMDGPU.def +++ b/clang/include/clang/Basic/BuiltinsAMDGPU.def @@ -432,13 +432,15 @@ TARGET_BUILTIN(__builtin_amdgcn_s_wakeup_barrier, "vi", "n", "gfx12-insts") TARGET_BUILTIN(__builtin_amdgcn_s_barrier_leave, "b", "n", "gfx12-insts") TARGET_BUILTIN(__builtin_amdgcn_s_get_barrier_state, "Uii", "n", "gfx12-insts") -TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_v2i32, "V2iV2i*1", "nc", "gfx12-insts,wavefrontsize32") -TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_v8i16, "V8sV8s*1", "nc", "gfx12-insts,wavefrontsize32") -TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_v8f16, "V8hV8h*1", "nc", "gfx12-insts,wavefrontsize32") - -TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_i32, "ii*1", "nc", "gfx12-insts,wavefrontsize64") -TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_v4i16, "V4sV4s*1", "nc", "gfx12-insts,wavefrontsize64") -TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_v4f16, "V4hV4h*1", "nc", "gfx12-insts,wavefrontsize64") +TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_b64_v2i32, "V2iV2i*1", "nc", "gfx12-insts,wavefrontsize32") +TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_b128_v8i16, "V8sV8s*1", "nc", "gfx12-insts,wavefrontsize32") +TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_b128_v8f16, "V8hV8h*1", "nc", "gfx12-insts,wavefrontsize32") +TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_b128_v8bf16, "V8yV8y*1", "nc", "gfx12-insts,wavefrontsize32") + +TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_b64_i32, "ii*1", "nc", "gfx12-insts,wavefrontsize64") +TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_b128_v4i16, "V4sV4s*1", "nc", "gfx12-insts,wavefrontsize64") +TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_b128_v4f16, "V4hV4h*1", "nc", "gfx12-insts,wavefrontsize64") +TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_b128_v4bf16, "V4yV4y*1", "nc", "gfx12-insts,wavefrontsize64") //===----------------------------------------------------------------------===// // WMMA builtins. diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp index e14e89088282..2eaceeba6177 100644 --- a/clang/lib/CodeGen/CGBuiltin.cpp +++ b/clang/lib/CodeGen/CGBuiltin.cpp @@ -18531,35 +18531,45 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID, llvm::Function *F = CGM.getIntrinsic(IID, {ArgTy}); return Builder.CreateCall(F, {Addr, Val, ZeroI32, ZeroI32, ZeroI1}); } - case AMDGPU::BI__builtin_amdgcn_global_load_tr_i32: - case AMDGPU::BI__builtin_amdgcn_global_load_tr_v2i32: - case AMDGPU::BI__builtin_amdgcn_global_load_tr_v4f16: - case AMDGPU::BI__builtin_amdgcn_global_load_tr_v4i16: - case AMDGPU::BI__builtin_amdgcn_global_load_tr_v8f16: - case AMDGPU::BI__builtin_amdgcn_global_load_tr_v8i16: { + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_i32: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_v2i32: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4bf16: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4f16: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4i16: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8bf16: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8f16: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8i16: { llvm::Type *ArgTy; switch (BuiltinID) { - case AMDGPU::BI__builtin_amdgcn_global_load_tr_i32: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_i32: ArgTy = llvm::Type::getInt32Ty(getLLVMContext()); break; - case AMDGPU::BI__builtin_amdgcn_global_load_tr_v2i32: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_v2i32: ArgTy = llvm::FixedVectorType::get( llvm::Type::getInt32Ty(getLLVMContext()), 2); break; - case AMDGPU::BI__builtin_amdgcn_global_load_tr_v4f16: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4bf16: + ArgTy = llvm::FixedVectorType::get( + llvm::Type::getBFloatTy(getLLVMContext()), 4); + break; + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4f16: ArgTy = llvm::FixedVectorType::get( llvm::Type::getHalfTy(getLLVMContext()), 4); break; - case AMDGPU::BI__builtin_amdgcn_global_load_tr_v4i16: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4i16: ArgTy = llvm::FixedVectorType::get( llvm::Type::getInt16Ty(getLLVMContext()), 4); break; - case AMDGPU::BI__builtin_amdgcn_global_load_tr_v8f16: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8bf16: + ArgTy = llvm::FixedVectorType::get( + llvm::Type::getBFloatTy(getLLVMContext()), 8); + break; + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8f16: ArgTy = llvm::FixedVectorType::get( llvm::Type::getHalfTy(getLLVMContext()), 8); break; - case AMDGPU::BI__builtin_amdgcn_global_load_tr_v8i16: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8i16: ArgTy = llvm::FixedVectorType::get( llvm::Type::getInt16Ty(getLLVMContext()), 8); break; diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx11-err.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx11-err.cl index f7afb7cb97ed..4363769b8645 100644 --- a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx11-err.cl +++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx11-err.cl @@ -6,21 +6,22 @@ typedef int v2i __attribute__((ext_vector_type(2))); typedef half v8h __attribute__((ext_vector_type(8))); typedef short v8s __attribute__((ext_vector_type(8))); +typedef __bf16 v8bf16 __attribute__((ext_vector_type(8))); typedef half v4h __attribute__((ext_vector_type(4))); typedef short v4s __attribute__((ext_vector_type(4))); +typedef __bf16 v4bf16 __attribute__((ext_vector_type(4))); - - -void amdgcn_global_load_tr(global v2i* v2i_inptr, global v8s* v8s_inptr, global v8h* v8h_inptr, - global int* int_inptr, global v4s* v4s_inptr, global v4h* v4h_inptr) +void amdgcn_global_load_tr(global v2i* v2i_inptr, global v8s* v8s_inptr, global v8h* v8h_inptr, global v8bf16* v8bf16_inptr, + global int* int_inptr, global v4s* v4s_inptr, global v4h* v4h_inptr, global v4bf16* v4bf16_inptr) { - v2i out_1 = __builtin_amdgcn_global_load_tr_v2i32(v2i_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_v2i32' needs target feature gfx12-insts,wavefrontsize32}} - v8s out_2 = __builtin_amdgcn_global_load_tr_v8i16(v8s_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_v8i16' needs target feature gfx12-insts,wavefrontsize32}} - v8h out_3 = __builtin_amdgcn_global_load_tr_v8f16(v8h_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_v8f16' needs target feature gfx12-insts,wavefrontsize32}} - - int out_4 = __builtin_amdgcn_global_load_tr_i32(int_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_i32' needs target feature gfx12-insts,wavefrontsize64}} - v4s out_5 = __builtin_amdgcn_global_load_tr_v4i16(v4s_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_v4i16' needs target feature gfx12-insts,wavefrontsize64}} - v4h out_6 = __builtin_amdgcn_global_load_tr_v4f16(v4h_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_v4f16' needs target feature gfx12-insts,wavefrontsize64}} + v2i out_1 = __builtin_amdgcn_global_load_tr_b64_v2i32(v2i_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b64_v2i32' needs target feature gfx12-insts,wavefrontsize32}} + v8s out_2 = __builtin_amdgcn_global_load_tr_b128_v8i16(v8s_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v8i16' needs target feature gfx12-insts,wavefrontsize32}} + v8h out_3 = __builtin_amdgcn_global_load_tr_b128_v8f16(v8h_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v8f16' needs target feature gfx12-insts,wavefrontsize32}} + v8bf16 o4 = __builtin_amdgcn_global_load_tr_b128_v8bf16(v8bf16_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v8bf16' needs target feature gfx12-insts,wavefrontsize32}} + + int out_5 = __builtin_amdgcn_global_load_tr_b64_i32(int_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b64_i32' needs target feature gfx12-insts,wavefrontsize64}} + v4s out_6 = __builtin_amdgcn_global_load_tr_b128_v4i16(v4s_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v4i16' needs target feature gfx12-insts,wavefrontsize64}} + v4h out_7 = __builtin_amdgcn_global_load_tr_b128_v4f16(v4h_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v4f16' needs target feature gfx12-insts,wavefrontsize64}} + v4bf16 o8 = __builtin_amdgcn_global_load_tr_b128_v4bf16(v4bf16_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v4bf16' needs target feature gfx12-insts,wavefrontsize64}} } - diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx12-w32-err.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx12-w32-err.cl index 04ac0a66db7c..208f92fc5d44 100644 --- a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx12-w32-err.cl +++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx12-w32-err.cl @@ -5,11 +5,12 @@ typedef half v4h __attribute__((ext_vector_type(4))); typedef short v4s __attribute__((ext_vector_type(4))); +typedef __bf16 v4bf16 __attribute__((ext_vector_type(4))); -void amdgcn_global_load_tr(global int* int_inptr, global v4s* v4s_inptr, global v4h* v4h_inptr) +void amdgcn_global_load_tr(global int* int_inptr, global v4s* v4s_inptr, global v4h* v4h_inptr, global v4bf16* v4bf16_inptr) { - int out_4 = __builtin_amdgcn_global_load_tr_i32(int_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_i32' needs target feature gfx12-insts,wavefrontsize64}} - v4s out_5 = __builtin_amdgcn_global_load_tr_v4i16(v4s_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_v4i16' needs target feature gfx12-insts,wavefrontsize64}} - v4h out_6 = __builtin_amdgcn_global_load_tr_v4f16(v4h_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_v4f16' needs target feature gfx12-insts,wavefrontsize64}} + int out_1 = __builtin_amdgcn_global_load_tr_b64_i32(int_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b64_i32' needs target feature gfx12-insts,wavefrontsize64}} + v4s out_2 = __builtin_amdgcn_global_load_tr_b128_v4i16(v4s_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v4i16' needs target feature gfx12-insts,wavefrontsize64}} + v4h out_3 = __builtin_amdgcn_global_load_tr_b128_v4f16(v4h_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v4f16' needs target feature gfx12-insts,wavefrontsize64}} + v4bf16 o4 = __builtin_amdgcn_global_load_tr_b128_v4bf16(v4bf16_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v4bf16' needs target feature gfx12-insts,wavefrontsize64}} } - diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx12-w64-err.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx12-w64-err.cl index 113b54b853a9..199146a9715d 100644 --- a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx12-w64-err.cl +++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx12-w64-err.cl @@ -6,11 +6,12 @@ typedef int v2i __attribute__((ext_vector_type(2))); typedef half v8h __attribute__((ext_vector_type(8))); typedef short v8s __attribute__((ext_vector_type(8))); +typedef __bf16 v8bf16 __attribute__((ext_vector_type(8))); -void amdgcn_global_load_tr(global v2i* v2i_inptr, global v8s* v8s_inptr, global v8h* v8h_inptr) +void amdgcn_global_load_tr(global v2i* v2i_inptr, global v8s* v8s_inptr, global v8h* v8h_inptr, global v8bf16* v8bf16_inptr) { - v2i out_1 = __builtin_amdgcn_global_load_tr_v2i32(v2i_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_v2i32' needs target feature gfx12-insts,wavefrontsize32}} - v8s out_2 = __builtin_amdgcn_global_load_tr_v8i16(v8s_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_v8i16' needs target feature gfx12-insts,wavefrontsize32}} - v8h out_3 = __builtin_amdgcn_global_load_tr_v8f16(v8h_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_v8f16' needs target feature gfx12-insts,wavefrontsize32}} + v2i out_1 = __builtin_amdgcn_global_load_tr_b64_v2i32(v2i_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b64_v2i32' needs target feature gfx12-insts,wavefrontsize32}} + v8s out_2 = __builtin_amdgcn_global_load_tr_b128_v8i16(v8s_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v8i16' needs target feature gfx12-insts,wavefrontsize32}} + v8h out_3 = __builtin_amdgcn_global_load_tr_b128_v8f16(v8h_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v8f16' needs target feature gfx12-insts,wavefrontsize32}} + v8bf16 o4 = __builtin_amdgcn_global_load_tr_b128_v8bf16(v8bf16_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v8bf16' needs target feature gfx12-insts,wavefrontsize32}} } - diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-w32.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-w32.cl index b5fcad68a470..0035b16b902b 100644 --- a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-w32.cl +++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-w32.cl @@ -5,44 +5,44 @@ typedef int v2i __attribute__((ext_vector_type(2))); typedef half v8h __attribute__((ext_vector_type(8))); typedef short v8s __attribute__((ext_vector_type(8))); +typedef __bf16 v8bf16 __attribute__((ext_vector_type(8))); -// Wave32 - -// -// amdgcn_global_load_tr -// - -// CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_v2i32( +// CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_b64_v2i32( // CHECK-GFX1200-NEXT: entry: // CHECK-GFX1200-NEXT: [[TMP0:%.*]] = tail call <2 x i32> @llvm.amdgcn.global.load.tr.v2i32(ptr addrspace(1) [[INPTR:%.*]]) // CHECK-GFX1200-NEXT: ret <2 x i32> [[TMP0]] // -v2i test_amdgcn_global_load_tr_v2i32(global v2i* inptr) +v2i test_amdgcn_global_load_tr_b64_v2i32(global v2i* inptr) { - return __builtin_amdgcn_global_load_tr_v2i32(inptr); + return __builtin_amdgcn_global_load_tr_b64_v2i32(inptr); } -// -// amdgcn_global_load_tr -// - -// CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_v8i16( +// CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_b128_v8i16( // CHECK-GFX1200-NEXT: entry: // CHECK-GFX1200-NEXT: [[TMP0:%.*]] = tail call <8 x i16> @llvm.amdgcn.global.load.tr.v8i16(ptr addrspace(1) [[INPTR:%.*]]) // CHECK-GFX1200-NEXT: ret <8 x i16> [[TMP0]] // -v8s test_amdgcn_global_load_tr_v8i16(global v8s* inptr) +v8s test_amdgcn_global_load_tr_b128_v8i16(global v8s* inptr) { - return __builtin_amdgcn_global_load_tr_v8i16(inptr); + return __builtin_amdgcn_global_load_tr_b128_v8i16(inptr); } -// CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_v8f16( +// CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_b128_v8f16( // CHECK-GFX1200-NEXT: entry: // CHECK-GFX1200-NEXT: [[TMP0:%.*]] = tail call <8 x half> @llvm.amdgcn.global.load.tr.v8f16(ptr addrspace(1) [[INPTR:%.*]]) // CHECK-GFX1200-NEXT: ret <8 x half> [[TMP0]] // -v8h test_amdgcn_global_load_tr_v8f16(global v8h* inptr) +v8h test_amdgcn_global_load_tr_b128_v8f16(global v8h* inptr) { - return __builtin_amdgcn_global_load_tr_v8f16(inptr); + return __builtin_amdgcn_global_load_tr_b128_v8f16(inptr); } +// CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_b128_v8bf16( +// CHECK-GFX1200-NEXT: entry: +// CHECK-GFX1200-NEXT: [[TMP0:%.*]] = tail call <8 x bfloat> @llvm.amdgcn.global.load.tr.v8bf16(ptr addrspace(1) [[INPTR:%.*]]) +// CHECK-GFX1200-NEXT: ret <8 x bfloat> [[TMP0]] +// +v8bf16 test_amdgcn_global_load_tr_b128_v8bf16(global v8bf16* inptr) +{ + return __builtin_amdgcn_global_load_tr_b128_v8bf16(inptr); +} diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-w64.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-w64.cl index 9c48ac071b4d..6c025bb5a55a 100644 --- a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-w64.cl +++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-w64.cl @@ -4,44 +4,44 @@ typedef half v4h __attribute__((ext_vector_type(4))); typedef short v4s __attribute__((ext_vector_type(4))); +typedef __bf16 v4bf16 __attribute__((ext_vector_type(4))); -// Wave64 - -// -// amdgcn_global_load_tr -// - -// CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_i32( +// CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_b64_i32( // CHECK-GFX1200-NEXT: entry: // CHECK-GFX1200-NEXT: [[TMP0:%.*]] = tail call i32 @llvm.amdgcn.global.load.tr.i32(ptr addrspace(1) [[INPTR:%.*]]) // CHECK-GFX1200-NEXT: ret i32 [[TMP0]] // -int test_amdgcn_global_load_tr_i32(global int* inptr) +int test_amdgcn_global_load_tr_b64_i32(global int* inptr) { - return __builtin_amdgcn_global_load_tr_i32(inptr); + return __builtin_amdgcn_global_load_tr_b64_i32(inptr); } -// -// amdgcn_global_load_tr -// - -// CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_v4i16( +// CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_b128_v4i16( // CHECK-GFX1200-NEXT: entry: // CHECK-GFX1200-NEXT: [[TMP0:%.*]] = tail call <4 x i16> @llvm.amdgcn.global.load.tr.v4i16(ptr addrspace(1) [[INPTR:%.*]]) // CHECK-GFX1200-NEXT: ret <4 x i16> [[TMP0]] // -v4s test_amdgcn_global_load_tr_v4i16(global v4s* inptr) +v4s test_amdgcn_global_load_tr_b128_v4i16(global v4s* inptr) { - return __builtin_amdgcn_global_load_tr_v4i16(inptr); + return __builtin_amdgcn_global_load_tr_b128_v4i16(inptr); } -// CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_v4f16( +// CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_b128_v4f16( // CHECK-GFX1200-NEXT: entry: // CHECK-GFX1200-NEXT: [[TMP0:%.*]] = tail call <4 x half> @llvm.amdgcn.global.load.tr.v4f16(ptr addrspace(1) [[INPTR:%.*]]) // CHECK-GFX1200-NEXT: ret <4 x half> [[TMP0]] // -v4h test_amdgcn_global_load_tr_v4f16(global v4h* inptr) +v4h test_amdgcn_global_load_tr_b128_v4f16(global v4h* inptr) { - return __builtin_amdgcn_global_load_tr_v4f16(inptr); + return __builtin_amdgcn_global_load_tr_b128_v4f16(inptr); } +// CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_b128_v4bf16( +// CHECK-GFX1200-NEXT: entry: +// CHECK-GFX1200-NEXT: [[TMP0:%.*]] = tail call <4 x bfloat> @llvm.amdgcn.global.load.tr.v4bf16(ptr addrspace(1) [[INPTR:%.*]]) +// CHECK-GFX1200-NEXT: ret <4 x bfloat> [[TMP0]] +// +v4bf16 test_amdgcn_global_load_tr_b128_v4bf16(global v4bf16* inptr) +{ + return __builtin_amdgcn_global_load_tr_b128_v4bf16(inptr); +} -- GitLab From 631e54aa1a0b7a79d0dec8dce7ec0f5e506acf6c Mon Sep 17 00:00:00 2001 From: "long.chen" Date: Fri, 22 Mar 2024 23:52:47 +0800 Subject: [PATCH 073/306] [mlir][arith] fix wrong floordivsi fold (#83248) Fixs https://github.com/llvm/llvm-project/issues/83079 --- mlir/lib/Dialect/Arith/IR/ArithOps.cpp | 36 ++------ .../Dialect/Arith/Transforms/ExpandOps.cpp | 58 ++++++------- mlir/test/Dialect/Arith/expand-ops.mlir | 84 ++++++++----------- .../Standard/CPU/test-ceil-floor-pos-neg.mlir | 30 +++++++ mlir/test/Transforms/canonicalize.mlir | 9 ++ 5 files changed, 102 insertions(+), 115 deletions(-) diff --git a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp index 9f64a07f31e3..2f32d9a26e77 100644 --- a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp +++ b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp @@ -689,43 +689,17 @@ OpFoldResult arith::FloorDivSIOp::fold(FoldAdaptor adaptor) { return getLhs(); // Don't fold if it would overflow or if it requires a division by zero. - bool overflowOrDiv0 = false; + bool overflowOrDiv = false; auto result = constFoldBinaryOp( adaptor.getOperands(), [&](APInt a, const APInt &b) { - if (overflowOrDiv0 || !b) { - overflowOrDiv0 = true; + if (b.isZero()) { + overflowOrDiv = true; return a; } - if (!a) - return a; - // After this point we know that neither a or b are zero. - unsigned bits = a.getBitWidth(); - APInt zero = APInt::getZero(bits); - bool aGtZero = a.sgt(zero); - bool bGtZero = b.sgt(zero); - if (aGtZero && bGtZero) { - // Both positive, return a / b. - return a.sdiv_ov(b, overflowOrDiv0); - } - if (!aGtZero && !bGtZero) { - // Both negative, return -a / -b. - APInt posA = zero.ssub_ov(a, overflowOrDiv0); - APInt posB = zero.ssub_ov(b, overflowOrDiv0); - return posA.sdiv_ov(posB, overflowOrDiv0); - } - if (!aGtZero && bGtZero) { - // A is negative, b is positive, return - ceil(-a, b). - APInt posA = zero.ssub_ov(a, overflowOrDiv0); - APInt ceil = signedCeilNonnegInputs(posA, b, overflowOrDiv0); - return zero.ssub_ov(ceil, overflowOrDiv0); - } - // A is positive, b is negative, return - ceil(a, -b). - APInt posB = zero.ssub_ov(b, overflowOrDiv0); - APInt ceil = signedCeilNonnegInputs(a, posB, overflowOrDiv0); - return zero.ssub_ov(ceil, overflowOrDiv0); + return a.sfloordiv_ov(b, overflowOrDiv); }); - return overflowOrDiv0 ? Attribute() : result; + return overflowOrDiv ? Attribute() : result; } //===----------------------------------------------------------------------===// diff --git a/mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp b/mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp index 7f246daf99ff..71e14a153cfd 100644 --- a/mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp +++ b/mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp @@ -110,9 +110,13 @@ struct CeilDivSIOpConverter : public OpRewritePattern { } }; -/// Expands FloorDivSIOp (n, m) into -/// 1) x = (m<0) ? 1 : -1 -/// 2) return (n*m<0) ? - ((-n+x) / m) -1 : n / m +/// Expands FloorDivSIOp (x, y) into +/// z = x / y +/// if (z * y != x && (x < 0) != (y < 0)) { +/// return z - 1; +/// } else { +/// return z; +/// } struct FloorDivSIOpConverter : public OpRewritePattern { using OpRewritePattern::OpRewritePattern; LogicalResult matchAndRewrite(arith::FloorDivSIOp op, @@ -121,41 +125,29 @@ struct FloorDivSIOpConverter : public OpRewritePattern { Type type = op.getType(); Value a = op.getLhs(); Value b = op.getRhs(); - Value plusOne = createConst(loc, type, 1, rewriter); + + Value quotient = rewriter.create(loc, a, b); + Value product = rewriter.create(loc, quotient, b); + Value notEqualDivisor = rewriter.create( + loc, arith::CmpIPredicate::ne, a, product); Value zero = createConst(loc, type, 0, rewriter); - Value minusOne = createConst(loc, type, -1, rewriter); - // Compute x = (b<0) ? 1 : -1. - Value compare = - rewriter.create(loc, arith::CmpIPredicate::slt, b, zero); - Value x = rewriter.create(loc, compare, plusOne, minusOne); - // Compute negative res: -1 - ((x-a)/b). - Value xMinusA = rewriter.create(loc, x, a); - Value xMinusADivB = rewriter.create(loc, xMinusA, b); - Value negRes = rewriter.create(loc, minusOne, xMinusADivB); - // Compute positive res: a/b. - Value posRes = rewriter.create(loc, a, b); - // Result is (a*b<0) ? negative result : positive result. - // Note, we want to avoid using a*b because of possible overflow. - // The case that matters are a>0, a==0, a<0, b>0 and b<0. We do - // not particuliarly care if a*b<0 is true or false when b is zero - // as this will result in an illegal divide. So `a*b<0` can be reformulated - // as `(a>0 && b<0) || (a>0 && b<0)' or `(a>0 && b<0) || (a>0 && b<=0)'. - // We pick the first expression here. + Value aNeg = rewriter.create(loc, arith::CmpIPredicate::slt, a, zero); - Value aPos = - rewriter.create(loc, arith::CmpIPredicate::sgt, a, zero); Value bNeg = rewriter.create(loc, arith::CmpIPredicate::slt, b, zero); - Value bPos = - rewriter.create(loc, arith::CmpIPredicate::sgt, b, zero); - Value firstTerm = rewriter.create(loc, aNeg, bPos); - Value secondTerm = rewriter.create(loc, aPos, bNeg); - Value compareRes = - rewriter.create(loc, firstTerm, secondTerm); - // Perform substitution and return success. - rewriter.replaceOpWithNewOp(op, compareRes, negRes, - posRes); + + Value signOpposite = rewriter.create( + loc, arith::CmpIPredicate::ne, aNeg, bNeg); + Value cond = + rewriter.create(loc, notEqualDivisor, signOpposite); + + Value minusOne = createConst(loc, type, -1, rewriter); + Value quotientMinusOne = + rewriter.create(loc, quotient, minusOne); + + rewriter.replaceOpWithNewOp(op, cond, quotientMinusOne, + quotient); return success(); } }; diff --git a/mlir/test/Dialect/Arith/expand-ops.mlir b/mlir/test/Dialect/Arith/expand-ops.mlir index 91f652e5a270..6bed93e4c969 100644 --- a/mlir/test/Dialect/Arith/expand-ops.mlir +++ b/mlir/test/Dialect/Arith/expand-ops.mlir @@ -66,23 +66,17 @@ func.func @ceildivi_index(%arg0: index, %arg1: index) -> (index) { func.func @floordivi(%arg0: i32, %arg1: i32) -> (i32) { %res = arith.floordivsi %arg0, %arg1 : i32 return %res : i32 -// CHECK: [[ONE:%.+]] = arith.constant 1 : i32 -// CHECK: [[ZERO:%.+]] = arith.constant 0 : i32 -// CHECK: [[MIN1:%.+]] = arith.constant -1 : i32 -// CHECK: [[CMP1:%.+]] = arith.cmpi slt, [[ARG1]], [[ZERO]] : i32 -// CHECK: [[X:%.+]] = arith.select [[CMP1]], [[ONE]], [[MIN1]] : i32 -// CHECK: [[TRUE1:%.+]] = arith.subi [[X]], [[ARG0]] : i32 -// CHECK: [[TRUE2:%.+]] = arith.divsi [[TRUE1]], [[ARG1]] : i32 -// CHECK: [[TRUE3:%.+]] = arith.subi [[MIN1]], [[TRUE2]] : i32 -// CHECK: [[FALSE:%.+]] = arith.divsi [[ARG0]], [[ARG1]] : i32 -// CHECK: [[NNEG:%.+]] = arith.cmpi slt, [[ARG0]], [[ZERO]] : i32 -// CHECK: [[NPOS:%.+]] = arith.cmpi sgt, [[ARG0]], [[ZERO]] : i32 -// CHECK: [[MNEG:%.+]] = arith.cmpi slt, [[ARG1]], [[ZERO]] : i32 -// CHECK: [[MPOS:%.+]] = arith.cmpi sgt, [[ARG1]], [[ZERO]] : i32 -// CHECK: [[TERM1:%.+]] = arith.andi [[NNEG]], [[MPOS]] : i1 -// CHECK: [[TERM2:%.+]] = arith.andi [[NPOS]], [[MNEG]] : i1 -// CHECK: [[CMP2:%.+]] = arith.ori [[TERM1]], [[TERM2]] : i1 -// CHECK: [[RES:%.+]] = arith.select [[CMP2]], [[TRUE3]], [[FALSE]] : i32 +// CHECK: %[[QUOTIENT:.*]] = arith.divsi %arg0, %arg1 : i32 +// CHECK: %[[PRODUCT:.*]] = arith.muli %[[QUOTIENT]], %arg1 : i32 +// CHECK: %[[NOT_EQ_PRODUCT:.*]] = arith.cmpi ne, %arg0, %[[PRODUCT]] : i32 +// CHECK-DAG: %[[ZERO:.*]] = arith.constant 0 : i32 +// CHECK: %[[NEG_DIVISOR:.*]] = arith.cmpi slt, %arg0, %[[ZERO]] : i32 +// CHECK: %[[NEG_DIVIDEND:.*]] = arith.cmpi slt, %arg1, %[[ZERO]] : i32 +// CHECK: %[[OPPOSITE_SIGN:.*]] = arith.cmpi ne, %[[NEG_DIVISOR]], %[[NEG_DIVIDEND]] : i1 +// CHECK: %[[CONDITION:.*]] = arith.andi %[[NOT_EQ_PRODUCT]], %[[OPPOSITE_SIGN]] : i1 +// CHECK-DAG: %[[NEG_ONE:.*]] = arith.constant -1 : i32 +// CHECK: %[[MINUS_ONE:.*]] = arith.addi %[[QUOTIENT]], %[[NEG_ONE]] : i32 +// CHECK: %[[RES:.*]] = arith.select %[[CONDITION]], %[[MINUS_ONE]], %[[QUOTIENT]] : i32 } // ----- @@ -93,23 +87,17 @@ func.func @floordivi(%arg0: i32, %arg1: i32) -> (i32) { func.func @floordivi_index(%arg0: index, %arg1: index) -> (index) { %res = arith.floordivsi %arg0, %arg1 : index return %res : index -// CHECK: [[ONE:%.+]] = arith.constant 1 : index -// CHECK: [[ZERO:%.+]] = arith.constant 0 : index -// CHECK: [[MIN1:%.+]] = arith.constant -1 : index -// CHECK: [[CMP1:%.+]] = arith.cmpi slt, [[ARG1]], [[ZERO]] : index -// CHECK: [[X:%.+]] = arith.select [[CMP1]], [[ONE]], [[MIN1]] : index -// CHECK: [[TRUE1:%.+]] = arith.subi [[X]], [[ARG0]] : index -// CHECK: [[TRUE2:%.+]] = arith.divsi [[TRUE1]], [[ARG1]] : index -// CHECK: [[TRUE3:%.+]] = arith.subi [[MIN1]], [[TRUE2]] : index -// CHECK: [[FALSE:%.+]] = arith.divsi [[ARG0]], [[ARG1]] : index -// CHECK: [[NNEG:%.+]] = arith.cmpi slt, [[ARG0]], [[ZERO]] : index -// CHECK: [[NPOS:%.+]] = arith.cmpi sgt, [[ARG0]], [[ZERO]] : index -// CHECK: [[MNEG:%.+]] = arith.cmpi slt, [[ARG1]], [[ZERO]] : index -// CHECK: [[MPOS:%.+]] = arith.cmpi sgt, [[ARG1]], [[ZERO]] : index -// CHECK: [[TERM1:%.+]] = arith.andi [[NNEG]], [[MPOS]] : i1 -// CHECK: [[TERM2:%.+]] = arith.andi [[NPOS]], [[MNEG]] : i1 -// CHECK: [[CMP2:%.+]] = arith.ori [[TERM1]], [[TERM2]] : i1 -// CHECK: [[RES:%.+]] = arith.select [[CMP2]], [[TRUE3]], [[FALSE]] : index +// CHECK: %[[QUOTIENT:.*]] = arith.divsi %arg0, %arg1 : index +// CHECK: %[[PRODUCT:.*]] = arith.muli %[[QUOTIENT]], %arg1 : index +// CHECK: %[[NOT_EQ_PRODUCT:.*]] = arith.cmpi ne, %arg0, %[[PRODUCT]] : index +// CHECK-DAG: %[[ZERO:.*]] = arith.constant 0 : index +// CHECK: %[[NEG_DIVISOR:.*]] = arith.cmpi slt, %arg0, %[[ZERO]] : index +// CHECK: %[[NEG_DIVIDEND:.*]] = arith.cmpi slt, %arg1, %[[ZERO]] : index +// CHECK: %[[OPPOSITE_SIGN:.*]] = arith.cmpi ne, %[[NEG_DIVISOR]], %[[NEG_DIVIDEND]] : i1 +// CHECK: %[[CONDITION:.*]] = arith.andi %[[NOT_EQ_PRODUCT]], %[[OPPOSITE_SIGN]] : i1 +// CHECK: %[[NEG_ONE:.*]] = arith.constant -1 : index +// CHECK-DAG: %[[MINUS_ONE:.*]] = arith.addi %[[QUOTIENT]], %[[NEG_ONE]] : index +// CHECK: %[[RES:.*]] = arith.select %[[CONDITION]], %[[MINUS_ONE]], %[[QUOTIENT]] : index } // ----- @@ -121,23 +109,17 @@ func.func @floordivi_index(%arg0: index, %arg1: index) -> (index) { func.func @floordivi_vec(%arg0: vector<4xi32>, %arg1: vector<4xi32>) -> (vector<4xi32>) { %res = arith.floordivsi %arg0, %arg1 : vector<4xi32> return %res : vector<4xi32> -// CHECK: %[[VAL_2:.*]] = arith.constant dense<1> : vector<4xi32> -// CHECK: %[[VAL_3:.*]] = arith.constant dense<0> : vector<4xi32> -// CHECK: %[[VAL_4:.*]] = arith.constant dense<-1> : vector<4xi32> -// CHECK: %[[VAL_5:.*]] = arith.cmpi slt, %[[VAL_1]], %[[VAL_3]] : vector<4xi32> -// CHECK: %[[VAL_6:.*]] = arith.select %[[VAL_5]], %[[VAL_2]], %[[VAL_4]] : vector<4xi1>, vector<4xi32> -// CHECK: %[[VAL_7:.*]] = arith.subi %[[VAL_6]], %[[VAL_0]] : vector<4xi32> -// CHECK: %[[VAL_8:.*]] = arith.divsi %[[VAL_7]], %[[VAL_1]] : vector<4xi32> -// CHECK: %[[VAL_9:.*]] = arith.subi %[[VAL_4]], %[[VAL_8]] : vector<4xi32> -// CHECK: %[[VAL_10:.*]] = arith.divsi %[[VAL_0]], %[[VAL_1]] : vector<4xi32> -// CHECK: %[[VAL_11:.*]] = arith.cmpi slt, %[[VAL_0]], %[[VAL_3]] : vector<4xi32> -// CHECK: %[[VAL_12:.*]] = arith.cmpi sgt, %[[VAL_0]], %[[VAL_3]] : vector<4xi32> -// CHECK: %[[VAL_13:.*]] = arith.cmpi slt, %[[VAL_1]], %[[VAL_3]] : vector<4xi32> -// CHECK: %[[VAL_14:.*]] = arith.cmpi sgt, %[[VAL_1]], %[[VAL_3]] : vector<4xi32> -// CHECK: %[[VAL_15:.*]] = arith.andi %[[VAL_11]], %[[VAL_14]] : vector<4xi1> -// CHECK: %[[VAL_16:.*]] = arith.andi %[[VAL_12]], %[[VAL_13]] : vector<4xi1> -// CHECK: %[[VAL_17:.*]] = arith.ori %[[VAL_15]], %[[VAL_16]] : vector<4xi1> -// CHECK: %[[VAL_18:.*]] = arith.select %[[VAL_17]], %[[VAL_9]], %[[VAL_10]] : vector<4xi1>, vector<4xi32> +// CHECK: %[[QUOTIENT:.*]] = arith.divsi %arg0, %arg1 : vector<4xi32> +// CHECK: %[[PRODUCT:.*]] = arith.muli %[[QUOTIENT]], %arg1 : vector<4xi32> +// CHECK: %[[NOT_EQ_PRODUCT:.*]] = arith.cmpi ne, %arg0, %[[PRODUCT]] : vector<4xi32> +// CHECK-DAG: %[[ZERO:.*]] = arith.constant dense<0> : vector<4xi32> +// CHECK: %[[NEG_DIVISOR:.*]] = arith.cmpi slt, %arg0, %[[ZERO]] : vector<4xi32> +// CHECK: %[[NEG_DIVIDEND:.*]] = arith.cmpi slt, %arg1, %[[ZERO]] : vector<4xi32> +// CHECK: %[[OPPOSITE_SIGN:.*]] = arith.cmpi ne, %[[NEG_DIVISOR]], %[[NEG_DIVIDEND]] : vector<4xi1> +// CHECK: %[[CONDITION:.*]] = arith.andi %[[NOT_EQ_PRODUCT]], %[[OPPOSITE_SIGN]] : vector<4xi1> +// CHECK-DAG: %[[NEG_ONE:.*]] = arith.constant dense<-1> : vector<4xi32> +// CHECK: %[[MINUS_ONE:.*]] = arith.addi %[[QUOTIENT]], %[[NEG_ONE]] : vector<4xi32> +// CHECK: %[[RES:.*]] = arith.select %[[CONDITION]], %[[MINUS_ONE]], %[[QUOTIENT]] : vector<4xi1>, vector<4xi32> } // ----- diff --git a/mlir/test/Integration/Dialect/Standard/CPU/test-ceil-floor-pos-neg.mlir b/mlir/test/Integration/Dialect/Standard/CPU/test-ceil-floor-pos-neg.mlir index 39fbb67512c6..a7013eacc984 100644 --- a/mlir/test/Integration/Dialect/Standard/CPU/test-ceil-floor-pos-neg.mlir +++ b/mlir/test/Integration/Dialect/Standard/CPU/test-ceil-floor-pos-neg.mlir @@ -2,6 +2,10 @@ // RUN: mlir-cpu-runner -e entry -entry-point-result=void \ // RUN: -shared-libs=%mlir_c_runner_utils | \ // RUN: FileCheck %s +// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf,lower-affine,convert-scf-to-cf,memref-expand,arith-expand),convert-vector-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ +// RUN: mlir-cpu-runner -e main -entry-point-result=void \ +// RUN: -shared-libs=%mlir_c_runner_utils | \ +// RUN: FileCheck %s --check-prefix=SCHECK func.func @transfer_read_2d(%A : memref<40xi32>, %base1: index) { %i42 = arith.constant -42: i32 @@ -101,3 +105,29 @@ func.func @entry() { // CHECK:( 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -2, -2, -2, -2, -2, -2, -2, -2, -2 ) // CHECK:( 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4 ) // CHECK:( 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ) + +// ----- + +func.func @non_inline_function() -> (i64, i64, i64, i64, i64, i64) { + %MIN_INT_MINUS_ONE = arith.constant -9223372036854775807 : i64 + %NEG_ONE = arith.constant -1 : i64 + %MIN_INT = arith.constant -9223372036854775808 : i64 + %ONE = arith.constant 1 : i64 + %MAX_INT = arith.constant 9223372036854775807 : i64 + return %MIN_INT_MINUS_ONE, %NEG_ONE, %MIN_INT, %ONE, %MAX_INT, %NEG_ONE : i64, i64, i64, i64, i64, i64 +} + +func.func @main() { + %0:6 = call @non_inline_function() : () -> (i64, i64, i64, i64, i64, i64) + %1 = arith.floordivsi %0#0, %0#1 : i64 + %2 = arith.floordivsi %0#2, %0#3 : i64 + %3 = arith.floordivsi %0#4, %0#5 : i64 + vector.print %1 : i64 + vector.print %2 : i64 + vector.print %3 : i64 + return +} + +// SCHECK: 9223372036854775807 +// SCHECK: -9223372036854775808 +// SCHECK: -9223372036854775807 diff --git a/mlir/test/Transforms/canonicalize.mlir b/mlir/test/Transforms/canonicalize.mlir index 2cf86b50d432..d2c2c12d3238 100644 --- a/mlir/test/Transforms/canonicalize.mlir +++ b/mlir/test/Transforms/canonicalize.mlir @@ -989,6 +989,15 @@ func.func @tensor_arith.floordivsi_by_one(%arg0: tensor<4x5xi32>) -> tensor<4x5x return %res : tensor<4x5xi32> } +// CHECK-LABEL: func @arith.floordivsi_by_one_overflow +func.func @arith.floordivsi_by_one_overflow() -> i64 { + %neg_one = arith.constant -1 : i64 + %min_int = arith.constant -9223372036854775808 : i64 + // CHECK: arith.floordivsi + %poision = arith.floordivsi %min_int, %neg_one : i64 + return %poision : i64 +} + // ----- // CHECK-LABEL: func @arith.ceildivsi_by_one -- GitLab From 01b1b0c1f728e2c2639edc654424f50830295989 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20Warzy=C5=84ski?= Date: Fri, 22 Mar 2024 15:53:04 +0000 Subject: [PATCH 074/306] [mlir][SVE] Add e2e for 1D depthwise WC convolution (#85225) Follow-up for https://github.com/llvm/llvm-project/pull/81625 --- .../Linalg/CPU/ArmSVE/1d-depthwise-conv.mlir | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/1d-depthwise-conv.mlir diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/1d-depthwise-conv.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/1d-depthwise-conv.mlir new file mode 100644 index 000000000000..57d69383c2de --- /dev/null +++ b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/1d-depthwise-conv.mlir @@ -0,0 +1,60 @@ +// DEFINE: %{compile} = mlir-opt %s \ +// DEFINE: -transform-interpreter -test-transform-dialect-erase-schedule \ +// DEFINE: -one-shot-bufferize="bufferize-function-boundaries" -lower-vector-mask -cse -canonicalize -convert-vector-to-scf -arm-sve-legalize-vector-storage \ +// DEFINE: -convert-vector-to-llvm="enable-arm-sve" -test-lower-to-llvm -o %t +// DEFINE: %{entry_point} = conv +// DEFINE: %{run} = %mcr_aarch64_cmd %t -e %{entry_point} -entry-point-result=void --march=aarch64 --mattr="+sve"\ +// DEFINE: -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils + +// RUN: %{compile} | %{run} | FileCheck %s + +func.func @conv() { + // Define input/output tensors + %input_init = tensor.empty() : tensor<1x8x6xi32> + %output_init = tensor.empty() : tensor<1x7x6xi32> + + %five = arith.constant 5 : i32 + %zero = arith.constant 0 : i32 + %input = linalg.fill ins(%five : i32) outs(%input_init : tensor<1x8x6xi32>) -> tensor<1x8x6xi32> + %output = linalg.fill ins(%zero : i32) outs(%output_init : tensor<1x7x6xi32>) -> tensor<1x7x6xi32> + + // Define the filter tensor + %filter = arith.constant dense<[ + [ 1, 2, 3, 4, 5, 6], + [ 11, 12, 13, 14, 15, 16] + ]> : tensor<2x6xi32> + + // static sizes -> dynamic sizes + %input_dyn = tensor.cast %input_init : tensor<1x8x6xi32> to tensor<1x8x?xi32> + %output_dyn = tensor.cast %output : tensor<1x7x6xi32> to tensor<1x7x?xi32> + %filter_dyn = tensor.cast %filter : tensor<2x6xi32> to tensor<2x?xi32> + + // Run the convolution + %res = linalg.depthwise_conv_1d_nwc_wc + ins(%input_dyn, %filter_dyn : tensor<1x8x?xi32>, tensor<2x?xi32>) + outs(%output_dyn : tensor<1x7x?xi32>) -> tensor<1x7x?xi32> + + // Print the results + // CHECK: SVE: START OF TEST OUTPUT + vector.print str "SVE: START OF TEST OUTPUT\n" + + // CHECK-NEXT: Unranked Memref base@ = {{.*}} rank = 3 offset = 0 sizes = [1, 7, 6] strides = [42, 6, 1] data = + // CHECK-COUNT-7: [60, 70, 80, 90, 100, 110] + %xf = tensor.cast %res : tensor<1x7x?xi32> to tensor<*xi32> + call @printMemrefI32(%xf) : (tensor<*xi32>) -> () + + // CHECK-NEXT: SVE: END OF TEST OUTPUT + vector.print str "SVE: END OF TEST OUTPUT\n" + + return +} + +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) { + %0 = transform.structured.match ops{["linalg.depthwise_conv_1d_nwc_wc"]} in %arg0 : (!transform.any_op) -> !transform.any_op + transform.structured.vectorize %0 vector_sizes [1, 7, [8], 2] : !transform.any_op + transform.yield + } +} + +func.func private @printMemrefI32(%ptr : tensor<*xi32>) attributes { llvm.emit_c_interface } -- GitLab From ab8ace3bfd5165a8532f710f9c2d8dd40c3fac39 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Fri, 22 Mar 2024 09:04:50 -0700 Subject: [PATCH 075/306] [bazel] Update to 7.x (#86297) --- utils/bazel/.bazelrc | 5 +++++ utils/bazel/.bazelversion | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/utils/bazel/.bazelrc b/utils/bazel/.bazelrc index c06e9b341626..46894decc7ad 100644 --- a/utils/bazel/.bazelrc +++ b/utils/bazel/.bazelrc @@ -6,6 +6,11 @@ # Common flags that apply to all configurations. # Use sparingly for things common to all compilers and platforms. ############################################################################### + +# Flip off to disable MODULE.bazel until we're ready. +# https://github.com/llvm/llvm-project/issues/55924 +common --enable_bzlmod=false + # Prevent invalid caching if input files are modified during a build. build --experimental_guard_against_concurrent_changes diff --git a/utils/bazel/.bazelversion b/utils/bazel/.bazelversion index 5e3254243a3b..21c8c7b46b89 100644 --- a/utils/bazel/.bazelversion +++ b/utils/bazel/.bazelversion @@ -1 +1 @@ -6.1.2 +7.1.1 -- GitLab From 26857582e5ee7980a71133ef8f8f579bcd90bdc8 Mon Sep 17 00:00:00 2001 From: Orlando Cazalet-Hyams Date: Fri, 22 Mar 2024 16:21:50 +0000 Subject: [PATCH 076/306] Revert "[RemoveDIs] Update DIBuilder C API with DbgRecord functions [2/2] (#85657)" This reverts commit 2091c74796b1dac68e622284c63a870b88b7554f. Builtbot failure: https://lab.llvm.org/buildbot/#/builders/16/builds/63080 --- llvm/docs/RemoveDIsDebugInfo.md | 11 +-- llvm/include/llvm-c/DebugInfo.h | 60 +++++--------- llvm/lib/IR/DebugInfo.cpp | 121 ++++++++++------------------- llvm/tools/llvm-c-test/debuginfo.c | 13 ++-- 4 files changed, 68 insertions(+), 137 deletions(-) diff --git a/llvm/docs/RemoveDIsDebugInfo.md b/llvm/docs/RemoveDIsDebugInfo.md index 9e50a2a604aa..a2f1e173d9d9 100644 --- a/llvm/docs/RemoveDIsDebugInfo.md +++ b/llvm/docs/RemoveDIsDebugInfo.md @@ -40,22 +40,15 @@ New functions (all to be deprecated) LLVMIsNewDbgInfoFormat # Returns true if the module is in the new non-instruction mode. LLVMSetIsNewDbgInfoFormat # Convert to the requested debug info format. -LLVMDIBuilderInsertDeclareIntrinsicBefore # Insert a debug intrinsic (old debug info format). +LLVMDIBuilderInsertDeclareIntrinsicBefore # Insert a debug intrinsic (old debug info format). LLVMDIBuilderInsertDeclareIntrinsicAtEnd # Same as above. LLVMDIBuilderInsertDbgValueIntrinsicBefore # Same as above. LLVMDIBuilderInsertDbgValueIntrinsicAtEnd # Same as above. -LLVMDIBuilderInsertDeclareRecordBefore # Insert a debug record (new debug info format). +LLVMDIBuilderInsertDeclareRecordBefore # Insert a debug record (new debug info format). LLVMDIBuilderInsertDeclareRecordAtEnd # Same as above. LLVMDIBuilderInsertDbgValueRecordBefore # Same as above. LLVMDIBuilderInsertDbgValueRecordAtEnd # Same as above. - -Existing functions (behaviour change) -------------------------------------- -LLVMDIBuilderInsertDeclareBefore # Insert a debug record (new debug info format) instead of a debug intrinsic (old debug info format). -LLVMDIBuilderInsertDeclareAtEnd # Same as above. -LLVMDIBuilderInsertDbgValueBefore # Same as above. -LLVMDIBuilderInsertDbgValueAtEnd # Same as above. ``` # Anything else? diff --git a/llvm/include/llvm-c/DebugInfo.h b/llvm/include/llvm-c/DebugInfo.h index dab1d697761b..b23ff63c862f 100644 --- a/llvm/include/llvm-c/DebugInfo.h +++ b/llvm/include/llvm-c/DebugInfo.h @@ -1249,12 +1249,7 @@ LLVMMetadataRef LLVMDIBuilderCreateTempGlobalVariableFwdDecl( LLVMMetadataRef Decl, uint32_t AlignInBits); /* - * Insert a new Declare DbgRecord before the given instruction. - * - * Only use in "new debug mode" (LLVMIsNewDbgInfoFormat() is true). - * Use LLVMSetIsNewDbgInfoFormat(LLVMBool) to convert between formats. - * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes - * + * Insert a new llvm.dbg.declare intrinsic call before the given instruction. * \param Builder The DIBuilder. * \param Storage The storage of the variable to declare. * \param VarInfo The variable's debug info descriptor. @@ -1262,13 +1257,13 @@ LLVMMetadataRef LLVMDIBuilderCreateTempGlobalVariableFwdDecl( * \param DebugLoc Debug info location. * \param Instr Instruction acting as a location for the new intrinsic. */ -LLVMDbgRecordRef +LLVMValueRef LLVMDIBuilderInsertDeclareBefore(LLVMDIBuilderRef Builder, LLVMValueRef Storage, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMValueRef Instr); /** * Soon to be deprecated. - * Only use in "old debug mode" (LLVMIsNewDbgInfoFormat() is false). + * Only use in "old debug mode" (LLVMIsNewDbgFormat() is false). * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes * * Insert a new llvm.dbg.declare intrinsic call before the given instruction. @@ -1284,7 +1279,7 @@ LLVMValueRef LLVMDIBuilderInsertDeclareIntrinsicBefore( LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMValueRef Instr); /** * Soon to be deprecated. - * Only use in "new debug mode" (LLVMIsNewDbgInfoFormat() is true). + * Only use in "new debug mode" (LLVMIsNewDbgFormat() is true). * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes * * Insert a Declare DbgRecord before the given instruction. @@ -1300,14 +1295,9 @@ LLVMDbgRecordRef LLVMDIBuilderInsertDeclareRecordBefore( LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMValueRef Instr); /** - * Insert a new Declare DbgRecord at the end of the given basic block. If the - * basic block has a terminator instruction, the intrinsic is inserted before - * that terminator instruction. - * - * Only use in "new debug mode" (LLVMIsNewDbgInfoFormat() is true). - * Use LLVMSetIsNewDbgInfoFormat(LLVMBool) to convert between formats. - * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes - * + * Insert a new llvm.dbg.declare intrinsic call at the end of the given basic + * block. If the basic block has a terminator instruction, the intrinsic is + * inserted before that terminator instruction. * \param Builder The DIBuilder. * \param Storage The storage of the variable to declare. * \param VarInfo The variable's debug info descriptor. @@ -1315,12 +1305,12 @@ LLVMDbgRecordRef LLVMDIBuilderInsertDeclareRecordBefore( * \param DebugLoc Debug info location. * \param Block Basic block acting as a location for the new intrinsic. */ -LLVMDbgRecordRef LLVMDIBuilderInsertDeclareAtEnd( +LLVMValueRef LLVMDIBuilderInsertDeclareAtEnd( LLVMDIBuilderRef Builder, LLVMValueRef Storage, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMBasicBlockRef Block); /** * Soon to be deprecated. - * Only use in "old debug mode" (LLVMIsNewDbgInfoFormat() is false). + * Only use in "old debug mode" (LLVMIsNewDbgFormat() is false). * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes * * Insert a new llvm.dbg.declare intrinsic call at the end of the given basic @@ -1338,7 +1328,7 @@ LLVMValueRef LLVMDIBuilderInsertDeclareIntrinsicAtEnd( LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMBasicBlockRef Block); /** * Soon to be deprecated. - * Only use in "new debug mode" (LLVMIsNewDbgInfoFormat() is true). + * Only use in "new debug mode" (LLVMIsNewDbgFormat() is true). * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes * * Insert a Declare DbgRecord at the end of the given basic block. If the basic @@ -1356,12 +1346,7 @@ LLVMDbgRecordRef LLVMDIBuilderInsertDeclareRecordAtEnd( LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMBasicBlockRef Block); /** - * Insert a new Value DbgRecord before the given instruction. - * - * Only use in "new debug mode" (LLVMIsNewDbgInfoFormat() is true). - * Use LLVMSetIsNewDbgInfoFormat(LLVMBool) to convert between formats. - * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes - * + * Insert a new llvm.dbg.value intrinsic call before the given instruction. * \param Builder The DIBuilder. * \param Val The value of the variable. * \param VarInfo The variable's debug info descriptor. @@ -1369,13 +1354,13 @@ LLVMDbgRecordRef LLVMDIBuilderInsertDeclareRecordAtEnd( * \param DebugLoc Debug info location. * \param Instr Instruction acting as a location for the new intrinsic. */ -LLVMDbgRecordRef +LLVMValueRef LLVMDIBuilderInsertDbgValueBefore(LLVMDIBuilderRef Builder, LLVMValueRef Val, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMValueRef Instr); /** * Soon to be deprecated. - * Only use in "old debug mode" (LLVMIsNewDbgInfoFormat() is false). + * Only use in "old debug mode" (Module::IsNewDbgInfoFormat is false). * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes * * Insert a new llvm.dbg.value intrinsic call before the given instruction. @@ -1391,7 +1376,7 @@ LLVMValueRef LLVMDIBuilderInsertDbgValueIntrinsicBefore( LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMValueRef Instr); /** * Soon to be deprecated. - * Only use in "new debug mode" (LLVMIsNewDbgInfoFormat() is true). + * Only use in "new debug mode" (Module::IsNewDbgInfoFormat is true). * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes * * Insert a new llvm.dbg.value intrinsic call before the given instruction. @@ -1407,14 +1392,9 @@ LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueRecordBefore( LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMValueRef Instr); /** - * Insert a new Value DbgRecord at the end of the given basic block. If the - * basic block has a terminator instruction, the intrinsic is inserted before - * that terminator instruction. - * - * Only use in "new debug mode" (LLVMIsNewDbgInfoFormat() is true). - * Use LLVMSetIsNewDbgInfoFormat(LLVMBool) to convert between formats. - * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes - * + * Insert a new llvm.dbg.value intrinsic call at the end of the given basic + * block. If the basic block has a terminator instruction, the intrinsic is + * inserted before that terminator instruction. * \param Builder The DIBuilder. * \param Val The value of the variable. * \param VarInfo The variable's debug info descriptor. @@ -1422,12 +1402,12 @@ LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueRecordBefore( * \param DebugLoc Debug info location. * \param Block Basic block acting as a location for the new intrinsic. */ -LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueAtEnd( +LLVMValueRef LLVMDIBuilderInsertDbgValueAtEnd( LLVMDIBuilderRef Builder, LLVMValueRef Val, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMBasicBlockRef Block); /** * Soon to be deprecated. - * Only use in "old debug mode" (LLVMIsNewDbgInfoFormat() is false). + * Only use in "old debug mode" (Module::IsNewDbgInfoFormat is false). * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes * * Insert a new llvm.dbg.value intrinsic call at the end of the given basic @@ -1445,7 +1425,7 @@ LLVMValueRef LLVMDIBuilderInsertDbgValueIntrinsicAtEnd( LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMBasicBlockRef Block); /** * Soon to be deprecated. - * Only use in "new debug mode" (LLVMIsNewDbgInfoFormat() is true). + * Only use in "new debug mode" (Module::IsNewDbgInfoFormat is true). * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes * * Insert a new llvm.dbg.value intrinsic call at the end of the given basic diff --git a/llvm/lib/IR/DebugInfo.cpp b/llvm/lib/IR/DebugInfo.cpp index 4206162d1768..09bce9df1f33 100644 --- a/llvm/lib/IR/DebugInfo.cpp +++ b/llvm/lib/IR/DebugInfo.cpp @@ -1665,12 +1665,12 @@ LLVMMetadataRef LLVMDIBuilderCreateTempGlobalVariableFwdDecl( unwrapDI(Decl), nullptr, AlignInBits)); } -LLVMDbgRecordRef +LLVMValueRef LLVMDIBuilderInsertDeclareBefore(LLVMDIBuilderRef Builder, LLVMValueRef Storage, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DL, LLVMValueRef Instr) { - return LLVMDIBuilderInsertDeclareRecordBefore(Builder, Storage, VarInfo, Expr, - DL, Instr); + return LLVMDIBuilderInsertDeclareIntrinsicBefore(Builder, Storage, VarInfo, + Expr, DL, Instr); } LLVMValueRef LLVMDIBuilderInsertDeclareIntrinsicBefore( LLVMDIBuilderRef Builder, LLVMValueRef Storage, LLVMMetadataRef VarInfo, @@ -1679,38 +1679,27 @@ LLVMValueRef LLVMDIBuilderInsertDeclareIntrinsicBefore( unwrap(Storage), unwrap(VarInfo), unwrap(Expr), unwrap(DL), unwrap(Instr)); - // This assert will fail if the module is in the new debug info format. - // This function should only be called if the module is in the old - // debug info format. - // See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes, - // LLVMIsNewDbgInfoFormat, and LLVMSetIsNewDbgInfoFormat for more info. assert(isa(DbgInst) && - "Function unexpectedly in new debug info format"); + "Inserted a DbgRecord into function using old debug info mode"); return wrap(cast(DbgInst)); } LLVMDbgRecordRef LLVMDIBuilderInsertDeclareRecordBefore( LLVMDIBuilderRef Builder, LLVMValueRef Storage, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DL, LLVMValueRef Instr) { - DbgInstPtr DbgInst = unwrap(Builder)->insertDeclare( - unwrap(Storage), unwrap(VarInfo), - unwrap(Expr), unwrap(DL), - unwrap(Instr)); - // This assert will fail if the module is in the old debug info format. - // This function should only be called if the module is in the new - // debug info format. - // See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes, - // LLVMIsNewDbgInfoFormat, and LLVMSetIsNewDbgInfoFormat for more info. - assert(isa(DbgInst) && - "Function unexpectedly in old debug info format"); - return wrap(cast(DbgInst)); + return wrap( + unwrap(Builder) + ->insertDeclare(unwrap(Storage), unwrap(VarInfo), + unwrap(Expr), unwrap(DL), + unwrap(Instr)) + .get()); } -LLVMDbgRecordRef +LLVMValueRef LLVMDIBuilderInsertDeclareAtEnd(LLVMDIBuilderRef Builder, LLVMValueRef Storage, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DL, LLVMBasicBlockRef Block) { - return LLVMDIBuilderInsertDeclareRecordAtEnd(Builder, Storage, VarInfo, Expr, - DL, Block); + return LLVMDIBuilderInsertDeclareIntrinsicAtEnd(Builder, Storage, VarInfo, + Expr, DL, Block); } LLVMValueRef LLVMDIBuilderInsertDeclareIntrinsicAtEnd( LLVMDIBuilderRef Builder, LLVMValueRef Storage, LLVMMetadataRef VarInfo, @@ -1718,36 +1707,26 @@ LLVMValueRef LLVMDIBuilderInsertDeclareIntrinsicAtEnd( DbgInstPtr DbgInst = unwrap(Builder)->insertDeclare( unwrap(Storage), unwrap(VarInfo), unwrap(Expr), unwrap(DL), unwrap(Block)); - // This assert will fail if the module is in the new debug info format. - // This function should only be called if the module is in the old - // debug info format. - // See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes, - // LLVMIsNewDbgInfoFormat, and LLVMSetIsNewDbgInfoFormat for more info. assert(isa(DbgInst) && - "Function unexpectedly in new debug info format"); + "Inserted a DbgRecord into function using old debug info mode"); return wrap(cast(DbgInst)); } LLVMDbgRecordRef LLVMDIBuilderInsertDeclareRecordAtEnd( LLVMDIBuilderRef Builder, LLVMValueRef Storage, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DL, LLVMBasicBlockRef Block) { - DbgInstPtr DbgInst = unwrap(Builder)->insertDeclare( - unwrap(Storage), unwrap(VarInfo), - unwrap(Expr), unwrap(DL), unwrap(Block)); - // This assert will fail if the module is in the old debug info format. - // This function should only be called if the module is in the new - // debug info format. - // See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes, - // LLVMIsNewDbgInfoFormat, and LLVMSetIsNewDbgInfoFormat for more info. - assert(isa(DbgInst) && - "Function unexpectedly in old debug info format"); - return wrap(cast(DbgInst)); + return wrap(unwrap(Builder) + ->insertDeclare(unwrap(Storage), + unwrap(VarInfo), + unwrap(Expr), + unwrap(DL), unwrap(Block)) + .get()); } -LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueBefore( +LLVMValueRef LLVMDIBuilderInsertDbgValueBefore( LLVMDIBuilderRef Builder, LLVMValueRef Val, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMValueRef Instr) { - return LLVMDIBuilderInsertDbgValueRecordBefore(Builder, Val, VarInfo, Expr, - DebugLoc, Instr); + return LLVMDIBuilderInsertDbgValueIntrinsicBefore(Builder, Val, VarInfo, Expr, + DebugLoc, Instr); } LLVMValueRef LLVMDIBuilderInsertDbgValueIntrinsicBefore( LLVMDIBuilderRef Builder, LLVMValueRef Val, LLVMMetadataRef VarInfo, @@ -1755,36 +1734,26 @@ LLVMValueRef LLVMDIBuilderInsertDbgValueIntrinsicBefore( DbgInstPtr DbgInst = unwrap(Builder)->insertDbgValueIntrinsic( unwrap(Val), unwrap(VarInfo), unwrap(Expr), unwrap(DebugLoc), unwrap(Instr)); - // This assert will fail if the module is in the new debug info format. - // This function should only be called if the module is in the old - // debug info format. - // See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes, - // LLVMIsNewDbgInfoFormat, and LLVMSetIsNewDbgInfoFormat for more info. assert(isa(DbgInst) && - "Function unexpectedly in new debug info format"); + "Inserted a DbgRecord into function using old debug info mode"); return wrap(cast(DbgInst)); } LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueRecordBefore( LLVMDIBuilderRef Builder, LLVMValueRef Val, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMValueRef Instr) { - DbgInstPtr DbgInst = unwrap(Builder)->insertDbgValueIntrinsic( - unwrap(Val), unwrap(VarInfo), unwrap(Expr), - unwrap(DebugLoc), unwrap(Instr)); - // This assert will fail if the module is in the old debug info format. - // This function should only be called if the module is in the new - // debug info format. - // See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes, - // LLVMIsNewDbgInfoFormat, and LLVMSetIsNewDbgInfoFormat for more info. - assert(isa(DbgInst) && - "Function unexpectedly in old debug info format"); - return wrap(cast(DbgInst)); + return wrap(unwrap(Builder) + ->insertDbgValueIntrinsic( + unwrap(Val), unwrap(VarInfo), + unwrap(Expr), unwrap(DebugLoc), + unwrap(Instr)) + .get()); } -LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueAtEnd( +LLVMValueRef LLVMDIBuilderInsertDbgValueAtEnd( LLVMDIBuilderRef Builder, LLVMValueRef Val, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMBasicBlockRef Block) { - return LLVMDIBuilderInsertDbgValueRecordAtEnd(Builder, Val, VarInfo, Expr, - DebugLoc, Block); + return LLVMDIBuilderInsertDbgValueIntrinsicAtEnd(Builder, Val, VarInfo, Expr, + DebugLoc, Block); } LLVMValueRef LLVMDIBuilderInsertDbgValueIntrinsicAtEnd( LLVMDIBuilderRef Builder, LLVMValueRef Val, LLVMMetadataRef VarInfo, @@ -1792,29 +1761,19 @@ LLVMValueRef LLVMDIBuilderInsertDbgValueIntrinsicAtEnd( DbgInstPtr DbgInst = unwrap(Builder)->insertDbgValueIntrinsic( unwrap(Val), unwrap(VarInfo), unwrap(Expr), unwrap(DebugLoc), unwrap(Block)); - // This assert will fail if the module is in the new debug info format. - // This function should only be called if the module is in the old - // debug info format. - // See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes, - // LLVMIsNewDbgInfoFormat, and LLVMSetIsNewDbgInfoFormat for more info. assert(isa(DbgInst) && - "Function unexpectedly in new debug info format"); + "Inserted a DbgRecord into function using old debug info mode"); return wrap(cast(DbgInst)); } LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueRecordAtEnd( LLVMDIBuilderRef Builder, LLVMValueRef Val, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMBasicBlockRef Block) { - DbgInstPtr DbgInst = unwrap(Builder)->insertDbgValueIntrinsic( - unwrap(Val), unwrap(VarInfo), unwrap(Expr), - unwrap(DebugLoc), unwrap(Block)); - // This assert will fail if the module is in the old debug info format. - // This function should only be called if the module is in the new - // debug info format. - // See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes, - // LLVMIsNewDbgInfoFormat, and LLVMSetIsNewDbgInfoFormat for more info. - assert(isa(DbgInst) && - "Function unexpectedly in old debug info format"); - return wrap(cast(DbgInst)); + return wrap(unwrap(Builder) + ->insertDbgValueIntrinsic( + unwrap(Val), unwrap(VarInfo), + unwrap(Expr), unwrap(DebugLoc), + unwrap(Block)) + .get()); } LLVMMetadataRef LLVMDIBuilderCreateAutoVariable( diff --git a/llvm/tools/llvm-c-test/debuginfo.c b/llvm/tools/llvm-c-test/debuginfo.c index 9b5c37b05d90..78ccaf12a380 100644 --- a/llvm/tools/llvm-c-test/debuginfo.c +++ b/llvm/tools/llvm-c-test/debuginfo.c @@ -136,13 +136,12 @@ int llvm_test_dibuilder(bool NewDebugInfoFormat) { LLVMMetadataRef FooParamVar1 = LLVMDIBuilderCreateParameterVariable(DIB, FunctionMetadata, "a", 1, 1, File, 42, Int64Ty, true, 0); - if (LLVMIsNewDbgInfoFormat(M)) - LLVMDIBuilderInsertDeclareAtEnd( + LLVMDIBuilderInsertDeclareRecordAtEnd( DIB, LLVMConstInt(LLVMInt64Type(), 0, false), FooParamVar1, FooParamExpression, FooParamLocation, FooEntryBlock); else - LLVMDIBuilderInsertDeclareIntrinsicAtEnd( + LLVMDIBuilderInsertDeclareAtEnd( DIB, LLVMConstInt(LLVMInt64Type(), 0, false), FooParamVar1, FooParamExpression, FooParamLocation, FooEntryBlock); LLVMMetadataRef FooParamVar2 = @@ -150,11 +149,11 @@ int llvm_test_dibuilder(bool NewDebugInfoFormat) { 42, Int64Ty, true, 0); if (LLVMIsNewDbgInfoFormat(M)) - LLVMDIBuilderInsertDeclareAtEnd( + LLVMDIBuilderInsertDeclareRecordAtEnd( DIB, LLVMConstInt(LLVMInt64Type(), 0, false), FooParamVar2, FooParamExpression, FooParamLocation, FooEntryBlock); else - LLVMDIBuilderInsertDeclareIntrinsicAtEnd( + LLVMDIBuilderInsertDeclareAtEnd( DIB, LLVMConstInt(LLVMInt64Type(), 0, false), FooParamVar2, FooParamExpression, FooParamLocation, FooEntryBlock); @@ -162,11 +161,11 @@ int llvm_test_dibuilder(bool NewDebugInfoFormat) { LLVMDIBuilderCreateParameterVariable(DIB, FunctionMetadata, "c", 1, 3, File, 42, VectorTy, true, 0); if (LLVMIsNewDbgInfoFormat(M)) - LLVMDIBuilderInsertDeclareAtEnd( + LLVMDIBuilderInsertDeclareRecordAtEnd( DIB, LLVMConstInt(LLVMInt64Type(), 0, false), FooParamVar3, FooParamExpression, FooParamLocation, FooEntryBlock); else - LLVMDIBuilderInsertDeclareIntrinsicAtEnd( + LLVMDIBuilderInsertDeclareAtEnd( DIB, LLVMConstInt(LLVMInt64Type(), 0, false), FooParamVar3, FooParamExpression, FooParamLocation, FooEntryBlock); -- GitLab From 8155ec13968b6457c61b8507f2ae8ba3ac3b748b Mon Sep 17 00:00:00 2001 From: "Yaxun (Sam) Liu" Date: Fri, 22 Mar 2024 12:30:02 -0400 Subject: [PATCH 077/306] [HIP][NFC] Refactor managed var codegen (#85976) Refactor managed variable handling in codegen so that the transformation is done separately from registration. This will allow the new driver to register the managed var in the linker wrapper. --- clang/lib/CodeGen/CGCUDANV.cpp | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/clang/lib/CodeGen/CGCUDANV.cpp b/clang/lib/CodeGen/CGCUDANV.cpp index d3f2573fd5e3..b756318c46a9 100644 --- a/clang/lib/CodeGen/CGCUDANV.cpp +++ b/clang/lib/CodeGen/CGCUDANV.cpp @@ -605,20 +605,10 @@ llvm::Function *CGNVCUDARuntime::makeRegisterGlobalsFn() { uint64_t VarSize = CGM.getDataLayout().getTypeAllocSize(Var->getValueType()); if (Info.Flags.isManaged()) { - auto *ManagedVar = new llvm::GlobalVariable( - CGM.getModule(), Var->getType(), - /*isConstant=*/false, Var->getLinkage(), - /*Init=*/Var->isDeclaration() - ? nullptr - : llvm::ConstantPointerNull::get(Var->getType()), - /*Name=*/"", /*InsertBefore=*/nullptr, - llvm::GlobalVariable::NotThreadLocal); - ManagedVar->setDSOLocal(Var->isDSOLocal()); - ManagedVar->setVisibility(Var->getVisibility()); - ManagedVar->setExternallyInitialized(true); - ManagedVar->takeName(Var); - Var->setName(Twine(ManagedVar->getName() + ".managed")); - replaceManagedVar(Var, ManagedVar); + assert(Var->getName().ends_with(".managed") && + "HIP managed variables not transformed"); + auto *ManagedVar = CGM.getModule().getNamedGlobal( + Var->getName().drop_back(StringRef(".managed").size())); llvm::Value *Args[] = { &GpuBinaryHandlePtr, ManagedVar, @@ -1093,7 +1083,9 @@ void CGNVCUDARuntime::transformManagedVars() { : llvm::ConstantPointerNull::get(Var->getType()), /*Name=*/"", /*InsertBefore=*/nullptr, llvm::GlobalVariable::NotThreadLocal, - CGM.getContext().getTargetAddressSpace(LangAS::cuda_device)); + CGM.getContext().getTargetAddressSpace(CGM.getLangOpts().CUDAIsDevice + ? LangAS::cuda_device + : LangAS::Default)); ManagedVar->setDSOLocal(Var->isDSOLocal()); ManagedVar->setVisibility(Var->getVisibility()); ManagedVar->setExternallyInitialized(true); @@ -1102,7 +1094,7 @@ void CGNVCUDARuntime::transformManagedVars() { Var->setName(Twine(ManagedVar->getName()) + ".managed"); // Keep managed variables even if they are not used in device code since // they need to be allocated by the runtime. - if (!Var->isDeclaration()) { + if (CGM.getLangOpts().CUDAIsDevice && !Var->isDeclaration()) { assert(!ManagedVar->isDeclaration()); CGM.addCompilerUsedGlobal(Var); CGM.addCompilerUsedGlobal(ManagedVar); @@ -1160,9 +1152,8 @@ void CGNVCUDARuntime::createOffloadingEntries() { // Returns module constructor to be added. llvm::Function *CGNVCUDARuntime::finalizeModule() { + transformManagedVars(); if (CGM.getLangOpts().CUDAIsDevice) { - transformManagedVars(); - // Mark ODR-used device variables as compiler used to prevent it from being // eliminated by optimization. This is necessary for device variables // ODR-used by host functions. Sema correctly marks them as ODR-used no -- GitLab From 7269570e4b2a5197201c959652c3e86804ed1eeb Mon Sep 17 00:00:00 2001 From: Abhin P Jose Date: Fri, 22 Mar 2024 17:39:43 +0100 Subject: [PATCH 078/306] Fixed build breaking due to #77178 and #86131 (#86290) Fixed a small issue of matching pthread signature, which was causing the build to break for the compiler-rt project after adding -Wcast-function-type-mismatch to -Wextra dignostic group (https://github.com/llvm/llvm-project/pull/77178 & https://github.com/llvm/llvm-project/pull/86131). --- compiler-rt/lib/asan/tests/asan_noinst_test.cpp | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/compiler-rt/lib/asan/tests/asan_noinst_test.cpp b/compiler-rt/lib/asan/tests/asan_noinst_test.cpp index 4c103609c83b..df7de2d7d15e 100644 --- a/compiler-rt/lib/asan/tests/asan_noinst_test.cpp +++ b/compiler-rt/lib/asan/tests/asan_noinst_test.cpp @@ -45,7 +45,8 @@ TEST(AddressSanitizer, InternalSimpleDeathTest) { EXPECT_DEATH(exit(1), ""); } -static void MallocStress(size_t n) { +static void *MallocStress(void *NumOfItrPtr) { + size_t n = *((size_t *)NumOfItrPtr); u32 seed = my_rand(); BufferedStackTrace stack1; stack1.trace_buffer[0] = 0xa123; @@ -90,20 +91,21 @@ static void MallocStress(size_t n) { } for (size_t i = 0; i < vec.size(); i++) __asan::asan_free(vec[i], &stack3, __asan::FROM_MALLOC); + return nullptr; } - TEST(AddressSanitizer, NoInstMallocTest) { - MallocStress(ASAN_LOW_MEMORY ? 300000 : 1000000); + const size_t kNumIterations = (ASAN_LOW_MEMORY) ? 300000 : 1000000; + MallocStress((void *)&kNumIterations); } TEST(AddressSanitizer, ThreadedMallocStressTest) { const int kNumThreads = 4; - const int kNumIterations = (ASAN_LOW_MEMORY) ? 10000 : 100000; + const size_t kNumIterations = (ASAN_LOW_MEMORY) ? 10000 : 100000; pthread_t t[kNumThreads]; for (int i = 0; i < kNumThreads; i++) { - PTHREAD_CREATE(&t[i], 0, (void* (*)(void *x))MallocStress, - (void*)kNumIterations); + PTHREAD_CREATE(&t[i], 0, (void *(*)(void *x))MallocStress, + (void *)&kNumIterations); } for (int i = 0; i < kNumThreads; i++) { PTHREAD_JOIN(t[i], 0); -- GitLab From d2f684685afeffcffba7e889e7267bce1d905911 Mon Sep 17 00:00:00 2001 From: Aaron Ballman Date: Fri, 22 Mar 2024 12:48:25 -0400 Subject: [PATCH 079/306] [C99] Update status of DR290, which we do not yet implement --- clang/test/C/drs/dr290.c | 20 ++++++++++++++++++++ clang/www/c_dr_status.html | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 clang/test/C/drs/dr290.c diff --git a/clang/test/C/drs/dr290.c b/clang/test/C/drs/dr290.c new file mode 100644 index 000000000000..3a6fd1d0dab6 --- /dev/null +++ b/clang/test/C/drs/dr290.c @@ -0,0 +1,20 @@ +/* RUN: %clang_cc1 -fsyntax-only -ast-dump %s | FileCheck %s + */ + +/* WG14 DR290: no + * FLT_EVAL_METHOD and extra precision and/or range + * + * We retain an implicit conversion based on the float eval method being used + * instead of dropping it due to the explicit cast. See GH86304 and C23 6.5.5p7. + */ + +#pragma clang fp eval_method(double) +_Static_assert((float)(123.0F * 2.0F) == (float)246.0F, ""); + +// CHECK: StaticAssertDecl +// CHECK-NEXT: ImplicitCastExpr {{.*}} '_Bool' +// CHECK-NEXT: BinaryOperator {{.*}} 'int' '==' +// NB: the following implicit cast is incorrect. +// CHECK-NEXT: ImplicitCastExpr {{.*}} 'double' FPEvalMethod=1 +// CHECK-NEXT: CStyleCastExpr {{.*}} 'float' FPEvalMethod=1 + diff --git a/clang/www/c_dr_status.html b/clang/www/c_dr_status.html index ed45123ffd0e..a41c4f717067 100644 --- a/clang/www/c_dr_status.html +++ b/clang/www/c_dr_status.html @@ -1686,7 +1686,7 @@ conformance.

290 C99 FLT_EVAL_METHOD and extra precision and/or range - Unknown + No 291 -- GitLab From 527a624205748814dd9309eda7ee308b40b2359a Mon Sep 17 00:00:00 2001 From: Aaron Ballman Date: Fri, 22 Mar 2024 12:49:12 -0400 Subject: [PATCH 080/306] [C11] Update the status of N1365 on constant expression handling This paper is about constant expression handling in the presence of FLT_EVAL_METHOD, which we handle via insertion of implicit cast nodes in the AST. --- clang/test/C/C11/n1365.c | 60 ++++++++++++++++++++++++++++++++++++++++ clang/www/c_status.html | 2 +- 2 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 clang/test/C/C11/n1365.c diff --git a/clang/test/C/C11/n1365.c b/clang/test/C/C11/n1365.c new file mode 100644 index 000000000000..3f769faa365d --- /dev/null +++ b/clang/test/C/C11/n1365.c @@ -0,0 +1,60 @@ +// RUN: %clang_cc1 -ast-dump %s | FileCheck %s + +/* WG14 N1365: Clang 16 + * Constant expressions + */ + +// Note: we don't allow you to expand __FLT_EVAL_METHOD__ in the presence of a +// pragma that changes its value. However, we can test that we have the correct +// constant expression behavior by testing that the AST has the correct implicit +// casts, which also specify that the cast was inserted due to an evaluation +// method requirement. +void func(void) { + { + #pragma clang fp eval_method(double) + _Static_assert(123.0F * 2.0F == 246.0F, ""); + // CHECK: StaticAssertDecl + // CHECK-NEXT: ImplicitCastExpr {{.*}} '_Bool' + // CHECK-NEXT: BinaryOperator {{.*}} 'int' '==' + // CHECK-NEXT: BinaryOperator {{.*}} 'double' '*' FPEvalMethod=1 + // CHECK-NEXT: ImplicitCastExpr {{.*}} 'double' FPEvalMethod=1 + // CHECK-NEXT: FloatingLiteral + // CHECK-NEXT: ImplicitCastExpr {{.*}} 'double' FPEvalMethod=1 + // CHECK-NEXT: FloatingLiteral + + // Ensure that a cast removes the extra precision. + _Static_assert((float)(123.0F * 2.0F) == 246.0F, ""); + // CHECK: StaticAssertDecl + // CHECK-NEXT: ImplicitCastExpr {{.*}} '_Bool' + // CHECK-NEXT: BinaryOperator {{.*}} 'int' '==' + // CHECK-NEXT: BinaryOperator {{.*}} 'double' '*' FPEvalMethod=1 + // CHECK-NEXT: ImplicitCastExpr {{.*}} 'double' FPEvalMethod=1 + // CHECK-NEXT: FloatingLiteral + // CHECK-NEXT: ImplicitCastExpr {{.*}} 'double' FPEvalMethod=1 + // CHECK-NEXT: FloatingLiteral + } + + { + #pragma clang fp eval_method(extended) + _Static_assert(123.0F * 2.0F == 246.0F, ""); + // CHECK: StaticAssertDecl + // CHECK-NEXT: ImplicitCastExpr {{.*}} '_Bool' + // CHECK-NEXT: BinaryOperator {{.*}} 'int' '==' + // CHECK-NEXT: BinaryOperator {{.*}} 'long double' '*' FPEvalMethod=2 + // CHECK-NEXT: ImplicitCastExpr {{.*}} 'long double' FPEvalMethod=2 + // CHECK-NEXT: FloatingLiteral + // CHECK-NEXT: ImplicitCastExpr {{.*}} 'long double' FPEvalMethod=2 + // CHECK-NEXT: FloatingLiteral + } + + { + #pragma clang fp eval_method(source) + _Static_assert(123.0F * 2.0F == 246.0F, ""); + // CHECK: StaticAssertDecl + // CHECK-NEXT: ImplicitCastExpr {{.*}} '_Bool' + // CHECK-NEXT: BinaryOperator {{.*}} 'int' '==' + // CHECK-NEXT: BinaryOperator {{.*}} 'float' '*' FPEvalMethod=0 + // CHECK-NEXT: FloatingLiteral + // CHECK-NEXT: FloatingLiteral + } +} diff --git a/clang/www/c_status.html b/clang/www/c_status.html index 0069da74cbd5..f00d5a6b7094 100644 --- a/clang/www/c_status.html +++ b/clang/www/c_status.html @@ -471,7 +471,7 @@ conformance.

Constant expressions N1365 - Unknown + Full Contractions and expression evaluation methods -- GitLab From 72c729f354d71697a1402720c90b57ff521b6739 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Fri, 22 Mar 2024 09:51:20 -0700 Subject: [PATCH 081/306] [bazel] Add support for --incompatible_disallow_empty_glob (#85999) This bazel flag, that should be flipped in an upcoming release https://github.com/bazelbuild/bazel/pull/15327, fails if globs have no matches. This helps find libraries where you are accidentally not including files because of typos. This change removes the various globs that were not matching anything, and uncovered some targets that were doing nothing because their source files were deleted. There are a few cases where globs were intentionally optional in the case of loops that expanded to different potential options, so those now use `allow_empty = True`. This allows downstream consumers to also flip this flags for their own builds, where previously this would fail in LLVM instead. The downside to this change is that if files are added in these relatively standard locations, manual work will have to be done to add this patterns back. If folks prefer we could instead add `allow_empty = True` to every glob. --- utils/bazel/.bazelrc | 4 + .../llvm-project-overlay/bolt/BUILD.bazel | 2 - .../clang-tools-extra/clang-tidy/defs.bzl | 4 +- .../llvm-project-overlay/clang/BUILD.bazel | 22 +- .../llvm-project-overlay/lld/BUILD.bazel | 2 - .../llvm-project-overlay/llvm/BUILD.bazel | 337 +++++++----------- .../llvm-project-overlay/mlir/BUILD.bazel | 144 -------- .../mlir/test/BUILD.bazel | 13 - .../mlir/unittests/BUILD.bazel | 38 -- 9 files changed, 138 insertions(+), 428 deletions(-) diff --git a/utils/bazel/.bazelrc b/utils/bazel/.bazelrc index 46894decc7ad..e8d055ec2322 100644 --- a/utils/bazel/.bazelrc +++ b/utils/bazel/.bazelrc @@ -35,6 +35,10 @@ build --features=layering_check # See: https://bazel.build/reference/be/functions#exports_files build --incompatible_no_implicit_file_export +# Enable so downstream users can flip this flag globally, this should +# eventually become the default +common --incompatible_disallow_empty_glob + ############################################################################### # Options to select different strategies for linking potential dependent # libraries. The default leaves it disabled. diff --git a/utils/bazel/llvm-project-overlay/bolt/BUILD.bazel b/utils/bazel/llvm-project-overlay/bolt/BUILD.bazel index 043a3b61a75f..1c12c8167ba4 100644 --- a/utils/bazel/llvm-project-overlay/bolt/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/bolt/BUILD.bazel @@ -221,8 +221,6 @@ cc_library( srcs = glob([ "lib/Target/AArch64/*.cpp", ]), - hdrs = glob([ - ]), includes = ["include"], deps = [ ":Core", diff --git a/utils/bazel/llvm-project-overlay/clang-tools-extra/clang-tidy/defs.bzl b/utils/bazel/llvm-project-overlay/clang-tools-extra/clang-tidy/defs.bzl index 41c03aad871c..5abe4b08f5d9 100644 --- a/utils/bazel/llvm-project-overlay/clang-tools-extra/clang-tidy/defs.bzl +++ b/utils/bazel/llvm-project-overlay/clang-tools-extra/clang-tidy/defs.bzl @@ -16,8 +16,8 @@ _common_library_deps = [ ] def clang_tidy_library(name, **kwargs): - kwargs["srcs"] = kwargs.get("srcs", native.glob([paths.join(name, "*.cpp")])) - kwargs["hdrs"] = kwargs.get("hdrs", native.glob([paths.join(name, "*.h")])) + kwargs["srcs"] = kwargs.get("srcs", native.glob([paths.join(name, "*.cpp")], allow_empty = True)) + kwargs["hdrs"] = kwargs.get("hdrs", native.glob([paths.join(name, "*.h")], allow_empty=True)) kwargs["deps"] = kwargs.get("deps", []) + _common_library_deps cc_library( name = name, diff --git a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel index 865cafbf50c6..c01986815afe 100644 --- a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel @@ -614,7 +614,6 @@ cc_library( "include/clang/Basic/Version.inc", ] + glob([ "lib/Basic/*.cpp", - "lib/Basic/*.c", "lib/Basic/*.h", "lib/Basic/Targets/*.cpp", "lib/Basic/Targets/*.h", @@ -1042,7 +1041,6 @@ cc_library( "lib/Analysis/FlowSensitive/Models/*.cpp", "lib/Analysis/FlowSensitive/*.cpp", "lib/Analysis/*.cpp", - "lib/Analysis/*.h", ]) + [ ":analysis_htmllogger_gen", ], @@ -1180,10 +1178,8 @@ gentbl( cc_library( name = "parse", - srcs = [ - ] + glob([ + srcs = glob([ "lib/Parse/*.cpp", - "lib/Parse/*.h", ]), hdrs = [ "include/clang/Parse/AttrParserStringSwitches.inc", @@ -1207,7 +1203,6 @@ cc_library( name = "ast_matchers", srcs = glob([ "lib/ASTMatchers/*.cpp", - "lib/ASTMatchers/*.h", ]), hdrs = glob(["include/clang/ASTMatchers/*.h"]), includes = ["include"], @@ -1241,7 +1236,6 @@ cc_library( name = "rewrite", srcs = glob([ "lib/Rewrite/*.cpp", - "lib/Rewrite/*.h", ]), hdrs = glob(["include/clang/Rewrite/Core/*.h"]), includes = ["include"], @@ -1275,7 +1269,6 @@ cc_library( name = "tooling_core", srcs = glob([ "lib/Tooling/Core/*.cpp", - "lib/Tooling/Core/*.h", ]), hdrs = glob(["include/clang/Tooling/Core/*.h"]), includes = ["include"], @@ -1340,11 +1333,9 @@ cc_library( name = "tooling_refactoring", srcs = glob([ "lib/Tooling/Refactoring/**/*.cpp", - "lib/Tooling/Refactoring/**/*.h", ]), hdrs = glob([ "include/clang/Tooling/Refactoring/**/*.h", - "include/clang/Tooling/Refactoring/**/*.def", ]), deps = [ ":ast", @@ -1593,9 +1584,6 @@ cc_library( srcs = glob( [ "lib/Driver/*.cpp", - "lib/Driver/*.h", - "lib/Driver/Arch/*.cpp", - "lib/Driver/Arch/*.h", "lib/Driver/ToolChains/*.cpp", "lib/Driver/ToolChains/*.h", "lib/Driver/ToolChains/Arch/*.cpp", @@ -1833,9 +1821,6 @@ cc_library( copts = ["$(STACK_FRAME_UNLIMITED)"], data = [":builtin_headers_gen"], includes = ["include"], - textual_hdrs = glob([ - "include/clang/Frontend/*.def", - ]), deps = [ ":apinotes", ":ast", @@ -1872,7 +1857,6 @@ cc_library( name = "frontend_rewrite", srcs = glob([ "lib/Frontend/Rewrite/*.cpp", - "lib/Frontend/Rewrite/*.h", ]), hdrs = glob(["include/clang/Rewrite/Frontend/*.h"]), includes = ["include"], @@ -2116,7 +2100,6 @@ cc_library( name = "frontend_tool", srcs = glob([ "lib/FrontendTool/*.cpp", - "lib/FrontendTool/*.h", ]), hdrs = glob(["include/clang/FrontendTool/*.h"]), includes = ["include"], @@ -2320,7 +2303,6 @@ cc_binary( testonly = 1, srcs = glob([ "tools/clang-import-test/*.cpp", - "tools/clang-import-test/*.h", ]), stamp = 0, deps = [ @@ -2350,7 +2332,6 @@ cc_library( name = "clang-driver", srcs = glob([ "tools/driver/*.cpp", - "tools/driver/*.h", ]) + ["clang-driver.cpp"], copts = [ # Disable stack frame size checks in the driver because @@ -2668,7 +2649,6 @@ cc_library( name = "extract_api", srcs = glob([ "lib/ExtractAPI/**/*.cpp", - "lib/ExtractAPI/**/*.h", ]), hdrs = glob(["include/clang/ExtractAPI/**/*.h"]), includes = ["include"], diff --git a/utils/bazel/llvm-project-overlay/lld/BUILD.bazel b/utils/bazel/llvm-project-overlay/lld/BUILD.bazel index 8fb71fc1f971..5a494a13acea 100644 --- a/utils/bazel/llvm-project-overlay/lld/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/lld/BUILD.bazel @@ -187,7 +187,6 @@ cc_library( name = "MinGW", srcs = glob([ "MinGW/*.cpp", - "MinGW/*.h", ]), includes = ["MinGW"], deps = [ @@ -296,7 +295,6 @@ cc_binary( name = "lld", srcs = glob([ "tools/lld/*.cpp", - "tools/lld/*.h", ]) + ["lld-driver.cpp"], deps = [ ":COFF", diff --git a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel index 07c5a00c07d7..f5a2d264b690 100644 --- a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel @@ -66,7 +66,10 @@ enum_targets_gen( llvm_target_asm_parsers = [ t for t in llvm_targets - if glob(["lib/Target/{}/AsmParser/CMakeLists.txt".format(t)]) + if glob( + ["lib/Target/{}/AsmParser/CMakeLists.txt".format(t)], + allow_empty = True, + ) ] enum_targets_gen( @@ -81,7 +84,10 @@ enum_targets_gen( llvm_target_disassemblers = [ t for t in llvm_targets - if glob(["lib/Target/{}/Disassembler/CMakeLists.txt".format(t)]) + if glob( + ["lib/Target/{}/Disassembler/CMakeLists.txt".format(t)], + allow_empty = True, + ) ] enum_targets_gen( @@ -96,7 +102,10 @@ enum_targets_gen( llvm_target_mcas = [ t for t in llvm_targets - if glob(["lib/Target/{}/MCA/CMakeLists.txt".format(t)]) + if glob( + ["lib/Target/{}/MCA/CMakeLists.txt".format(t)], + allow_empty = True, + ) ] enum_targets_gen( @@ -111,7 +120,10 @@ enum_targets_gen( llvm_target_exegesis = [ t for t in llvm_targets - if glob(["tools/llvm-exegesis/lib/{}/CMakeLists.txt".format(t)]) + if glob( + ["tools/llvm-exegesis/lib/{}/CMakeLists.txt".format(t)], + allow_empty = True, + ) ] enum_targets_gen( @@ -168,7 +180,6 @@ cc_library( name = "Demangle", srcs = glob([ "lib/Demangle/*.cpp", - "lib/Demangle/*.h", ]), hdrs = glob([ "include/llvm/Demangle/*.h", @@ -203,7 +214,6 @@ cc_library( "include/llvm/Option/*.h", ]) + select({ "@platforms//os:windows": glob([ - "lib/Support/Windows/*.h", "lib/Support/Windows/*.inc", ]), "//conditions:default": glob([ @@ -315,7 +325,6 @@ cc_library( name = "LineEditor", srcs = glob([ "lib/LineEditor/*.cpp", - "lib/LineEditor/*.h", ]), hdrs = glob(["include/llvm/LineEditor/*.h"]), copts = llvm_copts, @@ -329,7 +338,6 @@ cc_library( name = "Option", srcs = glob([ "lib/Option/*.cpp", - "lib/Option/*.h", ]), hdrs = glob(["include/llvm/Option/*.h"]), copts = llvm_copts, @@ -376,8 +384,6 @@ cc_library( name = "BinaryFormat", srcs = glob([ "lib/BinaryFormat/*.cpp", - "lib/BinaryFormat/*.def", - "lib/BinaryFormat/*.h", ]), hdrs = glob([ "include/llvm/BinaryFormat/*.h", @@ -409,7 +415,6 @@ cc_library( name = "DebugInfoMSF", srcs = glob([ "lib/DebugInfo/MSF/*.cpp", - "lib/DebugInfo/MSF/*.h", ]), hdrs = glob(["include/llvm/DebugInfo/MSF/*.h"]), copts = llvm_copts, @@ -420,7 +425,6 @@ cc_library( name = "DebugInfoBTF", srcs = glob([ "lib/DebugInfo/BTF/*.cpp", - "lib/DebugInfo/BTF/*.h", ]), hdrs = glob(["include/llvm/DebugInfo/BTF/*.h"]) + [ "include/llvm/DebugInfo/BTF/BTF.def", @@ -437,7 +441,6 @@ cc_library( name = "DebugInfoCodeView", srcs = glob([ "lib/DebugInfo/CodeView/*.cpp", - "lib/DebugInfo/CodeView/*.h", ]), hdrs = glob([ "include/llvm/DebugInfo/CodeView/*.h", @@ -480,9 +483,7 @@ cc_library( name = "DebugInfoPDB", srcs = glob([ "lib/DebugInfo/PDB/*.cpp", - "lib/DebugInfo/PDB/*.h", "lib/DebugInfo/PDB/Native/*.cpp", - "lib/DebugInfo/PDB/Native/*.h", ]), hdrs = glob([ "include/llvm/DebugInfo/PDB/*.h", @@ -523,12 +524,9 @@ cc_library( name = "MC", srcs = glob([ "lib/MC/*.cpp", - "lib/MC/*.h", ]), hdrs = glob([ "include/llvm/MC/*.h", - "include/llvm/MC/*.def", - "include/llvm/MC/*.inc", ]), copts = llvm_copts, deps = [ @@ -545,7 +543,6 @@ cc_library( name = "DebugInfoDWARF", srcs = glob([ "lib/DebugInfo/DWARF/*.cpp", - "lib/DebugInfo/DWARF/*.h", ]), hdrs = glob(["include/llvm/DebugInfo/DWARF/*.h"]), copts = llvm_copts, @@ -563,7 +560,6 @@ cc_library( name = "DebugInfoGSYM", srcs = glob([ "lib/DebugInfo/GSYM/*.cpp", - "lib/DebugInfo/GSYM/*.h", ]), hdrs = glob(["include/llvm/DebugInfo/GSYM/*.h"]), copts = llvm_copts, @@ -580,7 +576,6 @@ cc_library( name = "Symbolize", srcs = glob([ "lib/DebugInfo/Symbolize/*.cpp", - "lib/DebugInfo/Symbolize/*.h", ]), hdrs = glob([ "include/llvm/DebugInfo/Symbolize/*.h", @@ -658,7 +653,6 @@ cc_binary( srcs = glob( [ "utils/TableGen/*.cpp", - "utils/TableGen/*.inc", "utils/TableGen/*.h", "utils/TableGen/GlobalISel/*.cpp", "utils/TableGen/GlobalISel/*.h", @@ -821,7 +815,6 @@ cc_library( name = "BitstreamReader", srcs = glob([ "lib/Bitstream/Reader/*.cpp", - "lib/Bitstream/Reader/*.h", ]), hdrs = [ "include/llvm/Bitstream/BitCodeEnums.h", @@ -836,9 +829,6 @@ cc_library( cc_library( name = "BitstreamWriter", - srcs = glob([ - "lib/Bitstream/Writer/*.h", - ]), hdrs = [ "include/llvm/Bitstream/BitCodeEnums.h", "include/llvm/Bitstream/BitCodes.h", @@ -956,7 +946,6 @@ cc_library( name = "MCParser", srcs = glob([ "lib/MC/MCParser/*.cpp", - "lib/MC/MCParser/*.h", ]), hdrs = glob(["include/llvm/MC/MCParser/*.h"]), copts = llvm_copts, @@ -1002,9 +991,7 @@ cc_library( srcs = glob([ "lib/TextAPI/BinaryReader/**/*.cpp", ]), - hdrs = ["include/llvm/TextAPI/DylibReader.h"] + glob( - ["lib/TextAPI/BinaryReader/**/*.h"], - ), + hdrs = ["include/llvm/TextAPI/DylibReader.h"], copts = llvm_copts, deps = [ ":Object", @@ -1067,7 +1054,6 @@ cc_library( name = "ObjectYAML", srcs = glob([ "lib/ObjectYAML/*.cpp", - "lib/ObjectYAML/*.h", ]), hdrs = glob(["include/llvm/ObjectYAML/*.h"]), copts = llvm_copts, @@ -1085,7 +1071,6 @@ cc_library( name = "ProfileData", srcs = glob([ "lib/ProfileData/*.cpp", - "lib/ProfileData/*.h", ]), hdrs = glob([ "include/llvm/ProfileData/*.h", @@ -1109,7 +1094,6 @@ cc_library( name = "Coverage", srcs = glob([ "lib/ProfileData/Coverage/*.cpp", - "lib/ProfileData/Coverage/*.h", ]), hdrs = glob(["include/llvm/ProfileData/Coverage/*.h"]), copts = llvm_copts, @@ -1126,8 +1110,6 @@ cc_library( srcs = glob( [ "lib/Analysis/*.cpp", - "lib/Analysis/*.h", - "lib/Analysis/*.def", ], ), hdrs = glob( @@ -1185,7 +1167,6 @@ cc_library( name = "Target", srcs = glob([ "lib/Target/*.cpp", - "lib/Target/*.h", ]), hdrs = glob([ "include/llvm/Target/*.h", @@ -1221,14 +1202,11 @@ cc_library( name = "TargetParser", srcs = glob([ "lib/TargetParser/*.cpp", - "lib/TargetParser/*.h", ]) + select({ "@platforms//os:windows": glob([ - "lib/TargetParser/Windows/*.h", "lib/TargetParser/Windows/*.inc", ]), "//conditions:default": glob([ - "lib/TargetParser/Unix/*.h", "lib/TargetParser/Unix/*.inc", ]), }), @@ -1252,7 +1230,6 @@ cc_library( name = "DWP", srcs = glob([ "lib/DWP/*.cpp", - "lib/DWP/*.h", ]), hdrs = glob(["include/llvm/DWP/*.h"]), copts = llvm_copts, @@ -1269,7 +1246,6 @@ cc_library( name = "TransformUtils", srcs = glob([ "lib/Transforms/Utils/*.cpp", - "lib/Transforms/Utils/*.h", ]), hdrs = glob(["include/llvm/Transforms/Utils/*.h"]) + [ "include/llvm/Transforms/Utils.h", @@ -1390,7 +1366,6 @@ cc_library( name = "Scalar", srcs = glob([ "lib/Transforms/Scalar/*.cpp", - "lib/Transforms/Scalar/*.h", ]), hdrs = glob(["include/llvm/Transforms/Scalar/*.h"]) + [ "include/llvm/Transforms/Scalar.h", @@ -1432,9 +1407,6 @@ cc_library( cc_library( name = "FrontendDebug", - srcs = glob([ - "lib/Frontend/Debug/*.cpp", - ]), hdrs = glob([ "include/llvm/Frontend/Debug/*.h", ]), @@ -1530,8 +1502,6 @@ cc_library( ]), hdrs = glob([ "include/llvm/Frontend/OpenMP/*.h", - "include/llvm/Frontend/OpenMP/OMP/*.h", - "include/llvm/Frontend/*.h", ]) + [ "include/llvm/Frontend/OpenMP/OMP.h.inc", "include/llvm/Frontend/OpenMP/OMP.inc", @@ -1591,9 +1561,7 @@ cc_library( ]) + [ "include/llvm/Frontend/OpenACC/ACC.inc", ], - hdrs = glob([ - "include/llvm/Frontend/OpenACC/*.h", - ]) + ["include/llvm/Frontend/OpenACC/ACC.h.inc"], + hdrs = ["include/llvm/Frontend/OpenACC/ACC.h.inc"], copts = llvm_copts, deps = [ ":Analysis", @@ -1607,7 +1575,6 @@ cc_library( name = "AsmParser", srcs = glob([ "lib/AsmParser/*.cpp", - "lib/AsmParser/*.h", ]), hdrs = glob(["include/llvm/AsmParser/*.h"]), copts = llvm_copts, @@ -1623,7 +1590,6 @@ cc_library( name = "IRPrinter", srcs = glob([ "lib/IRPrinter/*.cpp", - "lib/IRPrinter/*.h", ]), hdrs = glob([ "include/llvm/IRPrinter/*.h", @@ -1640,7 +1606,6 @@ cc_library( name = "IRReader", srcs = glob([ "lib/IRReader/*.cpp", - "lib/IRReader/*.h", ]), hdrs = glob([ "include/llvm/IRReader/*.h", @@ -1683,7 +1648,6 @@ cc_library( name = "IPO", srcs = glob([ "lib/Transforms/IPO/*.cpp", - "lib/Transforms/IPO/*.h", ]), hdrs = glob([ "include/llvm/Transforms/IPO/*.h", @@ -1721,7 +1685,6 @@ cc_library( name = "CFGuard", srcs = glob([ "lib/Transforms/CFGuard/*.cpp", - "lib/Transforms/CFGuard/*.h", ]), hdrs = ["include/llvm/Transforms/CFGuard.h"], copts = llvm_copts, @@ -1736,7 +1699,6 @@ cc_library( name = "HipStdPar", srcs = glob([ "lib/Transforms/HipStdPar/*.cpp", - "lib/Transforms/HipStdPar/*.h", ]), hdrs = ["include/llvm/Transforms/HipStdPar/HipStdPar.h"], copts = llvm_copts, @@ -1826,7 +1788,6 @@ cc_library( copts = llvm_copts, textual_hdrs = glob([ "include/llvm/CodeGen/**/*.def", - "include/llvm/CodeGen/**/*.inc", ]), deps = [ ":AggressiveInstCombine", @@ -2305,10 +2266,13 @@ gentbl( td_file = "lib/Target/" + target["name"] + "/" + target["short_name"] + ".td", td_srcs = [ ":common_target_td_sources", - ] + glob([ - "lib/Target/" + target["name"] + "/*.td", - "lib/Target/" + target["name"] + "/GISel/*.td", - ]), + ] + glob( + [ + "lib/Target/" + target["name"] + "/*.td", + "lib/Target/" + target["name"] + "/GISel/*.td", + ], + allow_empty = True, + ), deps = target.get("tbl_deps", []), )], [cc_library( @@ -2332,43 +2296,49 @@ gentbl( # a number of targets due to crisscrossing inclusion of headers. [cc_library( name = target["name"] + "UtilsAndDesc", - srcs = glob([ - "lib/Target/" + target["name"] + "/MCTargetDesc/*.cpp", - "lib/Target/" + target["name"] + "/Utils/*.cpp", - - # We have to include these headers here as well as in the `hdrs` - # below to allow the `.cpp` files to use file-relative-inclusion to - # find them, even though consumers of this library use inclusion - # relative to the target with the `strip_includes_prefix` of this - # library. This mixture is likely incompatible with header modules. - "lib/Target/" + target["name"] + "/MCTargetDesc/*.h", - "lib/Target/" + target["name"] + "/Utils/*.h", - ]), - hdrs = glob([ - "lib/Target/" + target["name"] + "/MCTargetDesc/*.h", - "lib/Target/" + target["name"] + "/Utils/*.h", - - # This a bit of a hack to allow us to expose common, internal - # target header files to other libraries within the target via - # target-relative includes. This usage of headers is inherently - # non-modular as there is a mixture of target-relative inclusion - # using this rule and file-relative inclusion using the repeated - # listing of these headers in the `srcs` of subsequent rules. - "lib/Target/" + target["name"] + "/*.h", - - # FIXME: The entries below should be `textual_hdrs` instead of - # `hdrs`, but unfortunately that doesn't work with - # `strip_include_prefix`: - # https://github.com/bazelbuild/bazel/issues/12424 - # - # Once that issue is fixed and released, we can switch this to - # `textual_hdrs` and remove the feature disabling the various Bazel - # features (both current and under-development) that motivated the - # distinction between these two. - "lib/Target/" + target["name"] + "/*.def", - "lib/Target/" + target["name"] + "/*.inc", - "lib/Target/" + target["name"] + "/MCTargetDesc/*.def", - ]), + srcs = glob( + [ + "lib/Target/" + target["name"] + "/MCTargetDesc/*.cpp", + "lib/Target/" + target["name"] + "/Utils/*.cpp", + + # We have to include these headers here as well as in the `hdrs` + # below to allow the `.cpp` files to use file-relative-inclusion to + # find them, even though consumers of this library use inclusion + # relative to the target with the `strip_includes_prefix` of this + # library. This mixture is likely incompatible with header modules. + "lib/Target/" + target["name"] + "/MCTargetDesc/*.h", + "lib/Target/" + target["name"] + "/Utils/*.h", + ], + allow_empty = True, + ), + hdrs = glob( + [ + "lib/Target/" + target["name"] + "/MCTargetDesc/*.h", + "lib/Target/" + target["name"] + "/Utils/*.h", + + # This a bit of a hack to allow us to expose common, internal + # target header files to other libraries within the target via + # target-relative includes. This usage of headers is inherently + # non-modular as there is a mixture of target-relative inclusion + # using this rule and file-relative inclusion using the repeated + # listing of these headers in the `srcs` of subsequent rules. + "lib/Target/" + target["name"] + "/*.h", + + # FIXME: The entries below should be `textual_hdrs` instead of + # `hdrs`, but unfortunately that doesn't work with + # `strip_include_prefix`: + # https://github.com/bazelbuild/bazel/issues/12424 + # + # Once that issue is fixed and released, we can switch this to + # `textual_hdrs` and remove the feature disabling the various Bazel + # features (both current and under-development) that motivated the + # distinction between these two. + "lib/Target/" + target["name"] + "/*.def", + "lib/Target/" + target["name"] + "/*.inc", + "lib/Target/" + target["name"] + "/MCTargetDesc/*.def", + ], + allow_empty = True, + ), copts = llvm_copts, features = [ "-parse_headers", @@ -2392,20 +2362,26 @@ gentbl( )], [cc_library( name = target["name"] + "CodeGen", - srcs = glob([ - "lib/Target/" + target["name"] + "/GISel/*.cpp", - "lib/Target/" + target["name"] + "/GISel/*.h", - "lib/Target/" + target["name"] + "/*.cpp", - "lib/Target/" + target["name"] + "/*.h", - ]), + srcs = glob( + [ + "lib/Target/" + target["name"] + "/GISel/*.cpp", + "lib/Target/" + target["name"] + "/GISel/*.h", + "lib/Target/" + target["name"] + "/*.cpp", + "lib/Target/" + target["name"] + "/*.h", + ], + allow_empty = True, + ), hdrs = ["lib/Target/" + target["name"] + "/" + target["short_name"] + ".h"], copts = llvm_copts, features = ["-layering_check"], strip_include_prefix = "lib/Target/" + target["name"], - textual_hdrs = glob([ - "lib/Target/" + target["name"] + "/*.def", - "lib/Target/" + target["name"] + "/*.inc", - ]), + textual_hdrs = glob( + [ + "lib/Target/" + target["name"] + "/*.def", + "lib/Target/" + target["name"] + "/*.inc", + ], + allow_empty = True, + ), deps = [ ":Analysis", ":BinaryFormat", @@ -2430,10 +2406,13 @@ gentbl( )], [cc_library( name = target["name"] + "AsmParser", - srcs = glob([ - "lib/Target/" + target["name"] + "/AsmParser/*.cpp", - "lib/Target/" + target["name"] + "/AsmParser/*.h", - ]), + srcs = glob( + [ + "lib/Target/" + target["name"] + "/AsmParser/*.cpp", + "lib/Target/" + target["name"] + "/AsmParser/*.h", + ], + allow_empty = True, + ), copts = llvm_copts, deps = [ ":BinaryFormat", @@ -2464,9 +2443,12 @@ gentbl( # `textual_hdrs` and remove the feature disabling the various Bazel # features (both current and under-development) that motivated the # distinction between these two. - hdrs = glob([ - "lib/Target/" + target["name"] + "/Disassembler/*.h", - ]), + hdrs = glob( + [ + "lib/Target/" + target["name"] + "/Disassembler/*.h", + ], + allow_empty = True, + ), features = [ "-parse_headers", "-header_modules", @@ -2475,11 +2457,14 @@ gentbl( )], [cc_library( name = target["name"] + "Disassembler", - srcs = glob([ - "lib/Target/" + target["name"] + "/Disassembler/*.cpp", - "lib/Target/" + target["name"] + "/Disassembler/*.c", - "lib/Target/" + target["name"] + "/Disassembler/*.h", - ]), + srcs = glob( + [ + "lib/Target/" + target["name"] + "/Disassembler/*.cpp", + "lib/Target/" + target["name"] + "/Disassembler/*.c", + "lib/Target/" + target["name"] + "/Disassembler/*.h", + ], + allow_empty = True, + ), copts = llvm_copts, features = ["-layering_check"], deps = [ @@ -2497,11 +2482,14 @@ gentbl( )], [cc_library( name = target["name"] + "TargetMCA", - srcs = glob([ - "lib/Target/" + target["name"] + "/MCA/*.cpp", - "lib/Target/" + target["name"] + "/MCA/*.c", - "lib/Target/" + target["name"] + "/MCA/*.h", - ]), + srcs = glob( + [ + "lib/Target/" + target["name"] + "/MCA/*.cpp", + "lib/Target/" + target["name"] + "/MCA/*.c", + "lib/Target/" + target["name"] + "/MCA/*.h", + ], + allow_empty = True, + ), copts = llvm_copts, features = ["-layering_check"], deps = [ @@ -2559,28 +2547,10 @@ cc_library( textual_hdrs = ["lib/Passes/PassRegistry.def"], ) -cc_library( - name = "MLPolicies", - srcs = glob([ - "lib/Analysis/ML/*.cpp", - "lib/Analysis/ML/*.h", - ]), - hdrs = glob([ - "include/llvm/Analysis/ML/*.h", - ]), - copts = llvm_copts, - deps = [ - ":Analysis", - ":Core", - ":Support", - ], -) - cc_library( name = "Passes", srcs = glob([ "lib/Passes/*.cpp", - "lib/Passes/*.h", ]), hdrs = glob([ "include/llvm/Passes/*.h", @@ -2601,7 +2571,6 @@ cc_library( ":InstCombine", ":Instrumentation", ":MC", - ":MLPolicies", ":ObjCARC", ":Scalar", ":Support", @@ -2618,7 +2587,6 @@ cc_library( name = "LTO", srcs = glob([ "lib/LTO/*.cpp", - "lib/LTO/*.h", ]), hdrs = glob([ "include/llvm/LTO/*.h", @@ -2658,7 +2626,6 @@ cc_library( name = "ExecutionEngine", srcs = glob([ "lib/ExecutionEngine/*.cpp", - "lib/ExecutionEngine/*.h", "lib/ExecutionEngine/RuntimeDyld/*.cpp", "lib/ExecutionEngine/RuntimeDyld/*.h", "lib/ExecutionEngine/RuntimeDyld/Targets/*.cpp", @@ -2772,11 +2739,9 @@ cc_library( name = "OrcJIT", srcs = glob([ "lib/ExecutionEngine/Orc/*.cpp", - "lib/ExecutionEngine/Orc/*.h", ]), hdrs = glob([ "include/llvm/ExecutionEngine/Orc/*.h", - "include/llvm/ExecutionEngine/Orc/RPC/*.h", ]) + [ "include/llvm-c/LLJIT.h", "include/llvm-c/Orc.h", @@ -2900,7 +2865,6 @@ cc_library( name = "DWARFLinker", srcs = glob([ "lib/DWARFLinker/Classic/*.cpp", - "lib/DWARFLinker/Classic/*.h", ]), hdrs = glob(["include/llvm/DWARFLinker/Classic/*.h"]), copts = llvm_copts, @@ -2921,7 +2885,6 @@ cc_library( name = "DWARFLinkerBase", srcs = glob([ "lib/DWARFLinker/*.cpp", - "lib/DWARFLinker/*.h", ]), hdrs = glob(["include/llvm/DWARFLinker/*.h"]), copts = llvm_copts, @@ -3012,7 +2975,6 @@ cc_library( name = "InterfaceStub", srcs = glob([ "lib/InterfaceStub/*.cpp", - "lib/InterfaceStub/*.h", ]), hdrs = glob([ "include/llvm/InterfaceStub/*.h", @@ -3063,7 +3025,6 @@ cc_library( name = "MCA", srcs = glob([ "lib/MCA/**/*.cpp", - "lib/MCA/**/*.h", ]), hdrs = glob([ "include/llvm/MCA/**/*.h", @@ -3090,7 +3051,6 @@ cc_library( name = "XRay", srcs = glob([ "lib/XRay/*.cpp", - "lib/XRay/*.h", ]), hdrs = glob(["include/llvm/XRay/*.h"]), copts = llvm_copts, @@ -3147,21 +3107,24 @@ cc_library( cc_library( name = "Exegesis", - srcs = glob([ - "tools/llvm-exegesis/lib/*.cpp", - # We have to include these headers here as well as in the `hdrs` below - # to allow the `.cpp` files to use file-relative-inclusion to find - # them, even though consumers of this library use inclusion relative to - # `tools/llvm-exegesis/lib` with the `strip_includes_prefix` of this - # library. This mixture appears to be incompatible with header modules. - "tools/llvm-exegesis/lib/*.h", - ] + [ - "tools/llvm-exegesis/lib/{}/*.cpp".format(t) - for t in llvm_target_exegesis - ] + [ - "tools/llvm-exegesis/lib/{}/*.h".format(t) - for t in llvm_target_exegesis - ]), + srcs = glob( + [ + "tools/llvm-exegesis/lib/*.cpp", + # We have to include these headers here as well as in the `hdrs` below + # to allow the `.cpp` files to use file-relative-inclusion to find + # them, even though consumers of this library use inclusion relative to + # `tools/llvm-exegesis/lib` with the `strip_includes_prefix` of this + # library. This mixture appears to be incompatible with header modules. + "tools/llvm-exegesis/lib/*.h", + ] + [ + "tools/llvm-exegesis/lib/{}/*.cpp".format(t) + for t in llvm_target_exegesis + ] + [ + "tools/llvm-exegesis/lib/{}/*.h".format(t) + for t in llvm_target_exegesis + ], + allow_empty = True, + ), hdrs = glob(["tools/llvm-exegesis/lib/*.h"]), copts = llvm_copts, features = [ @@ -3335,7 +3298,6 @@ cc_binary( name = "llvm-ar", srcs = glob([ "tools/llvm-ar/*.cpp", - "tools/llvm-ar/*.h", ]) + ["llvm-ar-driver.cpp"], copts = llvm_copts, stamp = 0, @@ -3373,7 +3335,6 @@ cc_binary( name = "llvm-as", srcs = glob([ "tools/llvm-as/*.cpp", - "tools/llvm-as/*.h", ]), copts = llvm_copts, stamp = 0, @@ -3390,7 +3351,6 @@ cc_binary( name = "llvm-bcanalyzer", srcs = glob([ "tools/llvm-bcanalyzer/*.cpp", - "tools/llvm-bcanalyzer/*.h", ]), copts = llvm_copts, stamp = 0, @@ -3477,7 +3437,6 @@ cc_binary( name = "llvm-cvtres", srcs = glob([ "tools/llvm-cvtres/*.cpp", - "tools/llvm-cvtres/*.h", ]), copts = llvm_copts, stamp = 0, @@ -3511,7 +3470,6 @@ cc_binary( name = "llvm-cxxmap", srcs = glob([ "tools/llvm-cxxmap/*.cpp", - "tools/llvm-cxxmap/*.h", ]), copts = llvm_copts, stamp = 0, @@ -3546,7 +3504,6 @@ cc_binary( name = "llvm-cxxfilt", srcs = glob([ "tools/llvm-cxxfilt/*.cpp", - "tools/llvm-cxxfilt/*.h", ]) + ["llvm-cxxfilt-driver.cpp"], copts = llvm_copts, stamp = 0, @@ -3579,7 +3536,6 @@ cc_binary( name = "llvm-debuginfod-find", srcs = glob([ "tools/llvm-debuginfod-find/*.cpp", - "tools/llvm-debuginfod-find/*.h", ]), copts = llvm_copts, stamp = 0, @@ -3596,7 +3552,6 @@ cc_binary( name = "llvm-dis", srcs = glob([ "tools/llvm-dis/*.cpp", - "tools/llvm-dis/*.h", ]), copts = llvm_copts, stamp = 0, @@ -3691,7 +3646,6 @@ cc_binary( name = "llvm-dwp", srcs = glob([ "tools/llvm-dwp/*.cpp", - "tools/llvm-dwp/*.h", ]) + ["llvm-dwp-driver.cpp"], copts = llvm_copts, stamp = 0, @@ -3731,7 +3685,6 @@ cc_binary( name = "llvm-extract", srcs = glob([ "tools/llvm-extract/*.cpp", - "tools/llvm-extract/*.h", ]), copts = llvm_copts, stamp = 0, @@ -3773,7 +3726,6 @@ cc_binary( name = "llvm-gsymutil", srcs = glob([ "tools/llvm-gsymutil/*.cpp", - "tools/llvm-gsymutil/*.h", ]) + ["llvm-gsymutil-driver.cpp"], copts = llvm_copts, stamp = 0, @@ -3925,7 +3877,6 @@ cc_binary( name = "llvm-link", srcs = glob([ "tools/llvm-link/*.cpp", - "tools/llvm-link/*.h", ]), copts = llvm_copts, stamp = 0, @@ -3990,7 +3941,6 @@ cc_binary( name = "llvm-lto", srcs = glob([ "tools/llvm-lto/*.cpp", - "tools/llvm-lto/*.h", ]), copts = llvm_copts, stamp = 0, @@ -4013,7 +3963,6 @@ cc_binary( name = "llvm-lto2", srcs = glob([ "tools/llvm-lto2/*.cpp", - "tools/llvm-lto2/*.h", ]), copts = llvm_copts, stamp = 0, @@ -4159,7 +4108,6 @@ cc_binary( name = "llvm-mt", srcs = glob([ "tools/llvm-mt/*.cpp", - "tools/llvm-mt/*.h", ]) + ["llvm-mt-driver.cpp"], copts = llvm_copts, stamp = 0, @@ -4196,7 +4144,6 @@ cc_binary( name = "llvm-nm", srcs = glob([ "tools/llvm-nm/*.cpp", - "tools/llvm-nm/*.h", ]) + ["llvm-nm-driver.cpp"], copts = llvm_copts, stamp = 0, @@ -4280,7 +4227,6 @@ cc_binary( name = "llvm-stress", srcs = glob([ "tools/llvm-stress/*.cpp", - "tools/llvm-stress/*.h", ]), copts = llvm_copts, stamp = 0, @@ -4457,7 +4403,6 @@ cc_binary( name = "llvm-profdata", srcs = glob([ "tools/llvm-profdata/*.cpp", - "tools/llvm-profdata/*.h", ]) + ["llvm-profdata-driver.cpp"], copts = llvm_copts, stamp = 0, @@ -4641,7 +4586,6 @@ cc_binary( name = "llvm-rtdyld", srcs = glob([ "tools/llvm-rtdyld/*.cpp", - "tools/llvm-rtdyld/*.h", ]), copts = llvm_copts, stamp = 0, @@ -4683,7 +4627,6 @@ cc_binary( name = "llvm-size", srcs = glob([ "tools/llvm-size/*.cpp", - "tools/llvm-size/*.h", ]) + ["llvm-size-driver.cpp"], copts = llvm_copts, stamp = 0, @@ -4699,7 +4642,6 @@ cc_binary( name = "llvm-split", srcs = glob([ "tools/llvm-split/*.cpp", - "tools/llvm-split/*.h", ]), copts = llvm_copts, stamp = 0, @@ -4729,7 +4671,6 @@ cc_binary( name = "llvm-strings", srcs = glob([ "tools/llvm-strings/*.cpp", - "tools/llvm-strings/*.h", ]), copts = llvm_copts, stamp = 0, @@ -4766,7 +4707,6 @@ cc_binary( name = "llvm-symbolizer", srcs = glob([ "tools/llvm-symbolizer/*.cpp", - "tools/llvm-symbolizer/*.h", ]) + ["llvm-symbolizer-driver.cpp"], copts = llvm_copts, stamp = 0, @@ -4792,7 +4732,6 @@ cc_binary( name = "llvm-undname", srcs = glob([ "tools/llvm-undname/*.cpp", - "tools/llvm-undname/*.h", ]), copts = llvm_copts, stamp = 0, @@ -4806,7 +4745,6 @@ cc_binary( name = "llvm-xray", srcs = glob([ "tools/llvm-xray/*.cpp", - "tools/llvm-xray/*.cc", "tools/llvm-xray/*.h", ]), copts = llvm_copts, @@ -4889,7 +4827,6 @@ cc_binary( name = "sancov", srcs = glob([ "tools/sancov/*.cpp", - "tools/sancov/*.h", ]) + ["sancov-driver.cpp"], copts = llvm_copts, stamp = 0, @@ -4912,7 +4849,6 @@ cc_binary( name = "sanstats", srcs = glob([ "tools/sanstats/*.cpp", - "tools/sanstats/*.h", ]), copts = llvm_copts, stamp = 0, @@ -4927,7 +4863,6 @@ cc_binary( name = "split-file", srcs = glob([ "utils/split-file/*.cpp", - "utils/split-file/*.h", ]), copts = llvm_copts, stamp = 0, @@ -5023,7 +4958,6 @@ cc_library( testonly = True, srcs = glob([ "lib/Testing/Support/*.cpp", - "lib/Testing/Support/*.h", ]), hdrs = glob(["include/llvm/Testing/Support/*.h"]), copts = llvm_copts, @@ -5052,7 +4986,6 @@ cc_binary( testonly = True, srcs = glob([ "utils/FileCheck/*.cpp", - "utils/FileCheck/*.h", ]), copts = llvm_copts, stamp = 0, @@ -5098,7 +5031,6 @@ cc_binary( testonly = True, srcs = glob([ "utils/count/*.c", - "utils/count/*.h", ]), stamp = 0, deps = [":Support"], @@ -5109,7 +5041,6 @@ cc_binary( testonly = True, srcs = glob([ "tools/lli/ChildTarget/*.cpp", - "tools/lli/ChildTarget/*.h", ]), copts = llvm_copts, # The tests load code into this binary that expect to see symbols @@ -5176,7 +5107,6 @@ cc_binary( testonly = True, srcs = glob([ "tools/llvm-diff/*.cpp", - "tools/llvm-diff/*.h", ]), copts = llvm_copts, stamp = 0, @@ -5194,7 +5124,6 @@ cc_binary( testonly = True, srcs = glob([ "tools/llvm-isel-fuzzer/*.cpp", - "tools/llvm-isel-fuzzer/*.h", ]), copts = llvm_copts, stamp = 0, @@ -5240,7 +5169,6 @@ cc_binary( testonly = True, srcs = glob([ "utils/not/*.cpp", - "utils/not/*.h", ]), copts = llvm_copts, stamp = 0, @@ -5320,7 +5248,6 @@ cc_binary( testonly = True, srcs = glob([ "tools/llvm-tli-checker/*.cpp", - "tools/llvm-tli-checker/*.h", ]), copts = llvm_copts, stamp = 0, @@ -5367,7 +5294,6 @@ cc_binary( name = "verify-uselistorder", srcs = glob([ "tools/verify-uselistorder/*.cpp", - "tools/verify-uselistorder/*.h", ]), copts = llvm_copts, stamp = 0, @@ -5387,7 +5313,6 @@ cc_binary( testonly = True, srcs = glob([ "tools/yaml2obj/*.cpp", - "tools/yaml2obj/*.h", ]), copts = llvm_copts, stamp = 0, diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel index 3b575d4a413c..5b6e4678a05e 100644 --- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel @@ -325,9 +325,7 @@ cc_library( "lib/IR/*.cpp", "lib/IR/*.h", "lib/IR/PDL/*.cpp", - "lib/Bytecode/Reader/*.h", "lib/Bytecode/Writer/*.h", - "lib/Bytecode/*.h", ]) + [ "include/mlir/IR/PDLPatternMatch.h.inc", "include/mlir/Interfaces/CallInterfaces.h", @@ -1630,7 +1628,6 @@ cc_library( srcs = glob( [ "lib/Dialect/AMDGPU/Transforms/*.cpp", - "lib/Dialect/AMDGPU/Transforms/*.h", ], ), hdrs = glob(["include/mlir/Dialect/AMDGPU/Transforms/*.h"]), @@ -1769,7 +1766,6 @@ cc_library( name = "TargetCpp", srcs = glob([ "lib/Target/Cpp/*.cpp", - "lib/Target/Cpp/*.h", ]), hdrs = glob(["include/mlir/Target/Cpp/*.h"]), deps = [ @@ -1968,7 +1964,6 @@ cc_library( name = "ArmNeon2dToIntr", srcs = glob([ "lib/Conversion/ArmNeon2dToIntr/*.cpp", - "lib/Conversion/ArmNeon2dToIntr/*.h", ]), hdrs = glob([ "include/mlir/Conversion/ArmNeon2dToIntr/*.h", @@ -2840,7 +2835,6 @@ cc_library( name = "SCFTransforms", srcs = glob([ "lib/Dialect/SCF/Transforms/*.cpp", - "lib/Dialect/SCF/Transforms/*.h", ]), hdrs = glob([ "include/mlir/Dialect/SCF/Transforms/*.h", @@ -3189,7 +3183,6 @@ cc_library( name = "SparseTensorTransforms", srcs = glob([ "lib/Dialect/SparseTensor/Transforms/*.cpp", - "lib/Dialect/SparseTensor/Transforms/*.h", "lib/Dialect/SparseTensor/Transforms/Utils/*.cpp", "lib/Dialect/SparseTensor/Transforms/Utils/*.h", ]), @@ -3829,7 +3822,6 @@ cc_library( name = "Dialect", srcs = glob([ "lib/Dialect/*.cpp", - "lib/Dialect/*.h", ]), hdrs = glob([ "include/mlir/Dialect/*.h", @@ -3871,7 +3863,6 @@ cc_library( name = "DialectUtils", srcs = glob([ "lib/Dialect/Utils/*.cpp", - "lib/Dialect/Utils/*.h", ]), hdrs = glob([ "include/mlir/Dialect/Utils/*.h", @@ -3890,7 +3881,6 @@ cc_library( name = "AffineDialect", srcs = glob([ "lib/Dialect/Affine/IR/*.cpp", - "lib/Dialect/Affine/IR/*.h", ]), hdrs = glob([ "include/mlir/Dialect/Affine/IR/*.h", @@ -4012,7 +4002,6 @@ cc_library( name = "AffineAnalysis", srcs = glob([ "lib/Dialect/Affine/Analysis/*.cpp", - "lib/Dialect/Affine/Analysis/*.h", ]), hdrs = glob(["include/mlir/Dialect/Affine/Analysis/*.h"]), includes = ["include"], @@ -4036,7 +4025,6 @@ cc_library( srcs = glob( [ "lib/Dialect/Affine/Utils/*.cpp", - "lib/Dialect/Affine/Utils/*.h", ], ), hdrs = [ @@ -4083,7 +4071,6 @@ cc_library( name = "AffineTransforms", srcs = glob([ "lib/Dialect/Affine/Transforms/*.cpp", - "lib/Dialect/Affine/Transforms/*.h", ]), hdrs = [ "include/mlir/Dialect/Affine/Passes.h", @@ -4191,7 +4178,6 @@ cc_library( ":MemRefToSPIRV", ":NVGPUToNVVM", ":NVVMToLLVM", - ":OpenACCToLLVM", ":OpenACCToSCF", ":OpenMPToLLVM", ":PDLToPDLInterp", @@ -4224,7 +4210,6 @@ cc_library( name = "AsyncToLLVM", srcs = glob([ "lib/Conversion/AsyncToLLVM/*.cpp", - "lib/Conversion/AsyncToLLVM/*.h", ]), hdrs = glob(["include/mlir/Conversion/AsyncToLLVM/*.h"]), includes = ["include"], @@ -4250,7 +4235,6 @@ cc_library( name = "AffineToStandard", srcs = glob([ "lib/Conversion/AffineToStandard/*.cpp", - "lib/Conversion/AffineToStandard/*.h", ]), hdrs = glob(["include/mlir/Conversion/AffineToStandard/*.h"]), includes = ["include"], @@ -4271,32 +4255,11 @@ cc_library( ], ) -# SDBM dialect only contains attribute components that can be constructed given -# a dialect object, so whenever it is used it must also be registered. Therefore -# we don't split out the registration library for it. -cc_library( - name = "SDBM", - srcs = glob([ - "lib/Dialect/SDBM/*.cpp", - "lib/Dialect/SDBM/*.h", - ]), - hdrs = glob([ - "include/mlir/Dialect/SDBM/*.h", - ]), - includes = ["include"], - deps = [ - ":IR", - ":Support", - "//llvm:Support", - ], -) - cc_library( name = "SCFDialect", srcs = glob( [ "lib/Dialect/SCF/IR/*.cpp", - "lib/Dialect/SCF/IR/*.h", ], ), hdrs = glob( @@ -4605,7 +4568,6 @@ cc_library( name = "ShapeToStandard", srcs = glob([ "lib/Conversion/ShapeToStandard/*.cpp", - "lib/Conversion/ShapeToStandard/*.h", ]), hdrs = ["include/mlir/Conversion/ShapeToStandard/ShapeToStandard.h"], includes = ["include"], @@ -4645,7 +4607,6 @@ cc_library( name = "ShapeTransforms", srcs = glob([ "lib/Dialect/Shape/Transforms/*.cpp", - "lib/Dialect/Shape/Transforms/*.h", ]), hdrs = [ "include/mlir/Dialect/Shape/Analysis/ShapeMappingAnalysis.h", @@ -4728,7 +4689,6 @@ cc_library( srcs = glob( [ "lib/Dialect/ControlFlow/IR/*.cpp", - "lib/Dialect/ControlFlow/IR/*.h", ], ), hdrs = glob([ @@ -4774,13 +4734,10 @@ cc_library( srcs = glob( [ "lib/Dialect/Func/IR/*.cpp", - "lib/Dialect/Func/IR/*.h", - "lib/Dialect/Func/Utils/*.cpp", ], ), hdrs = glob([ "include/mlir/Dialect/Func/IR/*.h", - "include/mlir/Dialect/Func/Utils/*.h", ]), includes = ["include"], deps = [ @@ -4931,7 +4888,6 @@ cc_library( name = "FuncTransforms", srcs = glob([ "lib/Dialect/Func/Transforms/*.cpp", - "lib/Dialect/Func/Transforms/*.h", ]), hdrs = glob(["include/mlir/Dialect/Func/Transforms/*.h"]), includes = ["include"], @@ -5068,7 +5024,6 @@ cc_library( srcs = glob( [ "lib/Dialect/Vector/Transforms/*.cpp", - "lib/Dialect/Vector/Transforms/*.h", ], ), hdrs = glob([ @@ -5135,7 +5090,6 @@ cc_library( name = "Support", srcs = glob([ "lib/Support/*.cpp", - "lib/Support/*.h", ]), hdrs = glob(["include/mlir/Support/*.h"]), includes = ["include"], @@ -5149,11 +5103,8 @@ cc_library( name = "Debug", srcs = glob([ "lib/Debug/*.cpp", - "lib/Debug/*.h", "lib/Debug/BreakpointManagers/*.cpp", - "lib/Debug/BreakpointManagers/*.h", "lib/Debug/Observers/*.cpp", - "lib/Debug/Observers/*.h", ]), hdrs = glob([ "include/mlir/Debug/*.h", @@ -5195,8 +5146,6 @@ cc_library( [ "lib/Tools/mlir-lsp-server/*.cpp", "lib/Tools/mlir-lsp-server/*.h", - "lib/Tools/mlir-lsp-server/lsp/*.cpp", - "lib/Tools/mlir-lsp-server/lsp/*.h", ], ), hdrs = glob( @@ -5270,8 +5219,6 @@ cc_library( name = "BytecodeReader", srcs = glob([ "lib/Bytecode/Reader/*.cpp", - "lib/Bytecode/Reader/*.h", - "lib/Bytecode/*.h", ]), hdrs = glob([ "include/mlir/Bytecode/*.h", @@ -5291,7 +5238,6 @@ cc_library( srcs = glob([ "lib/Bytecode/Writer/*.cpp", "lib/Bytecode/Writer/*.h", - "lib/Bytecode/*.h", ]), hdrs = glob([ "include/mlir/Bytecode/*.h", @@ -5309,7 +5255,6 @@ cc_library( name = "Parser", srcs = glob([ "lib/Parser/*.cpp", - "lib/Parser/*.h", ]), hdrs = glob([ "include/mlir/Parser/*.h", @@ -5440,7 +5385,6 @@ cc_library( name = "LLVMIRTransforms", srcs = glob([ "lib/Dialect/LLVMIR/Transforms/*.cpp", - "lib/Dialect/LLVMIR/Transforms/*.h", ]), hdrs = glob(["include/mlir/Dialect/LLVMIR/Transforms/*.h"]), includes = ["include"], @@ -5593,7 +5537,6 @@ cc_library( srcs = glob( [ "lib/Dialect/GPU/IR/*.cpp", - "lib/Dialect/GPU/IR/*.h", ], ), hdrs = glob(["include/mlir/Dialect/GPU/IR/*.h"]), @@ -5687,7 +5630,6 @@ cc_library( srcs = glob( [ "lib/Dialect/GPU/Transforms/*.cpp", - "lib/Dialect/GPU/Transforms/*.h", ], ), hdrs = glob(["include/mlir/Dialect/GPU/Transforms/*.h"]), @@ -5877,7 +5819,6 @@ cc_library( name = "GPUToNVVMTransforms", srcs = glob([ "lib/Conversion/GPUToNVVM/*.cpp", - "lib/Conversion/GPUToNVVM/*.h", ]), hdrs = glob([ "include/mlir/Conversion/GPUToNVVM/*.h", @@ -5913,7 +5854,6 @@ cc_library( name = "AMDGPUToROCDL", srcs = glob([ "lib/Conversion/AMDGPUToROCDL/*.cpp", - "lib/Conversion/AMDGPUToROCDL/*.h", ]) + ["include/mlir/Conversion/GPUToROCDL/Runtimes.h"], hdrs = glob([ "include/mlir/Conversion/AMDGPUToROCDL/*.h", @@ -5938,7 +5878,6 @@ cc_library( name = "NVGPUToNVVM", srcs = glob([ "lib/Conversion/NVGPUToNVVM/*.cpp", - "lib/Conversion/NVGPUToNVVM/*.h", ]), hdrs = glob([ "include/mlir/Conversion/NVGPUToNVVM/*.h", @@ -5966,7 +5905,6 @@ cc_library( name = "VectorToSPIRV", srcs = glob([ "lib/Conversion/VectorToSPIRV/*.cpp", - "lib/Conversion/VectorToSPIRV/*.h", ]), hdrs = glob([ "include/mlir/Conversion/VectorToSPIRV/*.h", @@ -6095,7 +6033,6 @@ cc_library( name = "GPUToSPIRV", srcs = glob([ "lib/Conversion/GPUToSPIRV/*.cpp", - "lib/Conversion/GPUToSPIRV/*.h", ]), hdrs = glob([ "include/mlir/Conversion/GPUToSPIRV/*.h", @@ -6665,7 +6602,6 @@ cc_library( name = "PDLDialect", srcs = glob([ "lib/Dialect/PDL/IR/*.cpp", - "lib/Dialect/PDL/IR/*.h", ]), hdrs = glob([ "include/mlir/Dialect/PDL/IR/*.h", @@ -6740,7 +6676,6 @@ cc_library( name = "PDLInterpDialect", srcs = glob([ "lib/Dialect/PDLInterp/IR/*.cpp", - "lib/Dialect/PDLInterp/IR/*.h", ]), hdrs = glob([ "include/mlir/Dialect/PDLInterp/IR/*.h", @@ -7038,7 +6973,6 @@ cc_library( srcs = glob( [ "lib/Dialect/SPIRV/Transforms/*.cpp", - "lib/Dialect/SPIRV/Transforms/*.h", ], exclude = ["lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp"], ), @@ -7077,7 +7011,6 @@ cc_library( name = "MathToSPIRV", srcs = glob([ "lib/Conversion/MathToSPIRV/*.cpp", - "lib/Conversion/MathToSPIRV/*.h", ]), hdrs = glob([ "include/mlir/Conversion/MathToSPIRV/*.h", @@ -7104,7 +7037,6 @@ cc_library( name = "FuncToEmitC", srcs = glob([ "lib/Conversion/FuncToEmitC/*.cpp", - "lib/Conversion/FuncToEmitC/*.h", ]), hdrs = glob([ "include/mlir/Conversion/FuncToEmitC/*.h", @@ -7130,7 +7062,6 @@ cc_library( name = "FuncToSPIRV", srcs = glob([ "lib/Conversion/FuncToSPIRV/*.cpp", - "lib/Conversion/FuncToSPIRV/*.h", ]), hdrs = glob([ "include/mlir/Conversion/FuncToSPIRV/*.h", @@ -7162,7 +7093,6 @@ cc_library( name = "TensorToLinalg", srcs = glob([ "lib/Conversion/TensorToLinalg/*.cpp", - "lib/Conversion/TensorToLinalg/*.h", ]), hdrs = glob([ "include/mlir/Conversion/TensorToLinalg/*.h", @@ -7191,7 +7121,6 @@ cc_library( name = "TensorToSPIRV", srcs = glob([ "lib/Conversion/TensorToSPIRV/*.cpp", - "lib/Conversion/TensorToSPIRV/*.h", ]), hdrs = glob([ "include/mlir/Conversion/TensorToSPIRV/*.h", @@ -7477,7 +7406,6 @@ cc_library( srcs = glob( [ "lib/Dialect/Tensor/Transforms/*.cpp", - "lib/Dialect/Tensor/Transforms/*.h", ], ), hdrs = glob(["include/mlir/Dialect/Tensor/Transforms/*.h"]), @@ -7584,7 +7512,6 @@ cc_library( srcs = glob( include = [ "lib/Transforms/Utils/*.cpp", - "lib/Transforms/Utils/*.h", ], exclude = ["lib/Transforms/Utils/InliningUtils.cpp"], ), @@ -7923,7 +7850,6 @@ cc_library( name = "Transforms", srcs = glob([ "lib/Transforms/*.cpp", - "lib/Transforms/*.h", ]), hdrs = glob(["include/mlir/Transforms/*.h"]), includes = ["include"], @@ -8011,7 +7937,6 @@ cc_library( name = "SCFToSPIRV", srcs = glob([ "lib/Conversion/SCFToSPIRV/*.cpp", - "lib/Conversion/SCFToSPIRV/*.h", ]), hdrs = glob([ "include/mlir/Conversion/SCFToSPIRV/*.h", @@ -8261,7 +8186,6 @@ cc_library( name = "MemRefToEmitC", srcs = glob([ "lib/Conversion/MemRefToEmitC/*.cpp", - "lib/Conversion/MemRefToEmitC/*.h", ]), hdrs = glob([ "include/mlir/Conversion/MemRefToEmitC/*.h", @@ -8311,7 +8235,6 @@ cc_library( name = "MemRefToSPIRV", srcs = glob([ "lib/Conversion/MemRefToSPIRV/*.cpp", - "lib/Conversion/MemRefToSPIRV/*.h", ]), hdrs = glob([ "include/mlir/Conversion/MemRefToSPIRV/*.h", @@ -8371,7 +8294,6 @@ cc_library( name = "ArithToArmSME", srcs = glob([ "lib/Conversion/ArithToArmSME/*.cpp", - "lib/Conversion/ArithToArmSME/*.h", ]), hdrs = glob([ "include/mlir/Conversion/ArithToArmSME/*.h", @@ -8392,7 +8314,6 @@ cc_library( name = "ArithToEmitC", srcs = glob([ "lib/Conversion/ArithToEmitC/*.cpp", - "lib/Conversion/ArithToEmitC/*.h", ]), hdrs = glob([ "include/mlir/Conversion/ArithToEmitC/*.h", @@ -8683,9 +8604,7 @@ cc_library( srcs = glob( [ "lib/Analysis/*.cpp", - "lib/Analysis/*.h", "lib/Analysis/*/*.cpp", - "lib/Analysis/*/*.h", ], ), hdrs = glob( @@ -8991,7 +8910,6 @@ cc_library( ":IR", ":LLVMDialect", ":OpenACCDialect", - ":OpenACCToLLVM", ":OpenMPCommon", ":Support", ":ToLLVMIRTranslation", @@ -9357,7 +9275,6 @@ cc_library( ":SCFToGPU", ":SCFTransformOps", ":SCFTransforms", - ":SDBM", ":SPIRVDialect", ":SPIRVPassIncGen", ":SPIRVTarget", @@ -9468,7 +9385,6 @@ cc_binary( "//mlir/test:TestTosaDialect", "//mlir/test:TestTransformDialect", "//mlir/test:TestTransforms", - "//mlir/test:TestTypeDialect", "//mlir/test:TestVector", "//mlir/test:TestVectorToSPIRV", ], @@ -10086,7 +10002,6 @@ cc_library( srcs = glob( [ "lib/Dialect/OpenACC/IR/*.cpp", - "lib/Dialect/OpenACC/IR/*.h", ], ), hdrs = glob( @@ -10140,7 +10055,6 @@ cc_library( srcs = glob( [ "lib/Dialect/OpenACC/Transforms/*.cpp", - "lib/Dialect/OpenACC/Transforms/*.h", ], ), hdrs = glob(["include/mlir/Dialect/OpenACC/Transforms/*.h"]), @@ -10304,7 +10218,6 @@ cc_library( srcs = glob( [ "lib/Dialect/OpenMP/IR/*.cpp", - "lib/Dialect/OpenMP/IR/*.h", ], ), hdrs = glob( @@ -10340,7 +10253,6 @@ cc_library( name = "OpenACCToSCF", srcs = glob([ "lib/Conversion/OpenACCToSCF/*.cpp", - "lib/Conversion/OpenACCToSCF/*.h", ]), hdrs = glob([ "include/mlir/Conversion/OpenACCToSCF/*.h", @@ -10360,35 +10272,10 @@ cc_library( ], ) -cc_library( - name = "OpenACCToLLVM", - srcs = glob([ - "lib/Conversion/OpenACCToLLVM/*.cpp", - "lib/Conversion/OpenACCToLLVM/*.h", - ]), - hdrs = glob([ - "include/mlir/Conversion/OpenACCToLLVM/*.h", - ]), - includes = ["include"], - deps = [ - ":ConversionPassIncGen", - ":FuncDialect", - ":IR", - ":LLVMCommonConversion", - ":LLVMDialect", - ":OpenACCDialect", - ":Pass", - ":Transforms", - "//llvm:Core", - "//llvm:Support", - ], -) - cc_library( name = "OpenMPToLLVM", srcs = glob([ "lib/Conversion/OpenMPToLLVM/*.cpp", - "lib/Conversion/OpenMPToLLVM/*.h", ]), hdrs = glob([ "include/mlir/Conversion/OpenMPToLLVM/*.h", @@ -10598,7 +10485,6 @@ cc_library( name = "IndexToLLVM", srcs = glob([ "lib/Conversion/IndexToLLVM/*.cpp", - "lib/Conversion/IndexToLLVM/*.h", ]), hdrs = glob([ "include/mlir/Conversion/IndexToLLVM/*.h", @@ -10624,7 +10510,6 @@ cc_library( name = "IndexToSPIRV", srcs = glob([ "lib/Conversion/IndexToSPIRV/*.cpp", - "lib/Conversion/IndexToSPIRV/*.h", ]), hdrs = glob([ "include/mlir/Conversion/IndexToSPIRV/*.h", @@ -11015,7 +10900,6 @@ cc_library( name = "LinalgToStandard", srcs = glob([ "lib/Conversion/LinalgToStandard/*.cpp", - "lib/Conversion/LinalgToStandard/*.h", ]), hdrs = glob([ "include/mlir/Conversion/LinalgToStandard/*.h", @@ -11161,7 +11045,6 @@ cc_library( name = "LinalgUtils", srcs = glob([ "lib/Dialect/Linalg/Utils/*.cpp", - "lib/Dialect/Linalg/Utils/*.h", ]), hdrs = glob([ "include/mlir/Dialect/Linalg/Utils/*.h", @@ -11190,7 +11073,6 @@ cc_library( name = "LinalgTransforms", srcs = glob([ "lib/Dialect/Linalg/Transforms/*.cpp", - "lib/Dialect/Linalg/Transforms/*.h", ]), hdrs = [ "include/mlir/Dialect/Linalg/Passes.h", @@ -11542,7 +11424,6 @@ cc_library( name = "VectorToLLVM", srcs = glob([ "lib/Conversion/VectorToLLVM/*.cpp", - "lib/Conversion/VectorToLLVM/*.h", ]), hdrs = glob([ "include/mlir/Conversion/VectorToLLVM/*.h", @@ -11584,7 +11465,6 @@ cc_library( name = "VectorToArmSME", srcs = glob([ "lib/Conversion/VectorToArmSME/*.cpp", - "lib/Conversion/VectorToArmSME/*.h", ]), hdrs = glob([ "include/mlir/Conversion/VectorToArmSME/*.h", @@ -11605,7 +11485,6 @@ cc_library( name = "VectorToGPU", srcs = glob([ "lib/Conversion/VectorToGPU/*.cpp", - "lib/Conversion/VectorToGPU/*.h", ]), hdrs = glob([ "include/mlir/Conversion/VectorToGPU/*.h", @@ -11642,7 +11521,6 @@ cc_library( name = "VectorToSCF", srcs = glob([ "lib/Conversion/VectorToSCF/*.cpp", - "lib/Conversion/VectorToSCF/*.h", ]), hdrs = glob([ "include/mlir/Conversion/VectorToSCF/*.h", @@ -11792,7 +11670,6 @@ cc_library( name = "TosaDialect", srcs = glob([ "lib/Dialect/Tosa/IR/*.cpp", - "lib/Dialect/Tosa/IR/*.h", "lib/Dialect/Tosa/Utils/*.cpp", "lib/Dialect/Tosa/Transforms/*.cpp", ]), @@ -11832,7 +11709,6 @@ cc_library( name = "TosaToArith", srcs = glob([ "lib/Conversion/TosaToArith/*.cpp", - "lib/Conversion/TosaToArith/*.h", ]), hdrs = glob([ "include/mlir/Conversion/TosaToArith/*.h", @@ -11856,7 +11732,6 @@ cc_library( name = "TosaToLinalg", srcs = glob([ "lib/Conversion/TosaToLinalg/*.cpp", - "lib/Conversion/TosaToLinalg/*.h", ]), hdrs = glob([ "include/mlir/Conversion/TosaToLinalg/*.h", @@ -11889,7 +11764,6 @@ cc_library( name = "TosaToMLProgram", srcs = glob([ "lib/Conversion/TosaToMLProgram/*.cpp", - "lib/Conversion/TosaToMLProgram/*.h", ]), hdrs = glob([ "include/mlir/Conversion/TosaToMLProgram/*.h", @@ -11913,7 +11787,6 @@ cc_library( name = "TosaToSCF", srcs = glob([ "lib/Conversion/TosaToSCF/*.cpp", - "lib/Conversion/TosaToSCF/*.h", ]), hdrs = glob([ "include/mlir/Conversion/TosaToSCF/*.h", @@ -11938,7 +11811,6 @@ cc_library( name = "TosaToTensor", srcs = glob([ "lib/Conversion/TosaToTensor/*.cpp", - "lib/Conversion/TosaToTensor/*.h", ]), hdrs = glob([ "include/mlir/Conversion/TosaToTensor/*.h", @@ -12443,7 +12315,6 @@ cc_library( srcs = glob( [ "lib/Dialect/Complex/IR/*.cpp", - "lib/Dialect/Complex/IR/*.h", ], ), hdrs = ["include/mlir/Dialect/Complex/IR/Complex.h"], @@ -12466,7 +12337,6 @@ cc_library( name = "ComplexToLLVM", srcs = glob([ "lib/Conversion/ComplexToLLVM/*.cpp", - "lib/Conversion/ComplexToLLVM/*.h", ]), hdrs = glob([ "include/mlir/Conversion/ComplexToLLVM/*.h", @@ -12494,7 +12364,6 @@ cc_library( name = "ComplexToLibm", srcs = glob([ "lib/Conversion/ComplexToLibm/*.cpp", - "lib/Conversion/ComplexToLibm/*.h", ]), hdrs = glob([ "include/mlir/Conversion/ComplexToLibm/*.h", @@ -12518,7 +12387,6 @@ cc_library( name = "ComplexToSPIRV", srcs = glob([ "lib/Conversion/ComplexToSPIRV/*.cpp", - "lib/Conversion/ComplexToSPIRV/*.h", ]), hdrs = glob([ "include/mlir/Conversion/ComplexToSPIRV/*.h", @@ -12543,7 +12411,6 @@ cc_library( name = "ComplexToStandard", srcs = glob([ "lib/Conversion/ComplexToStandard/*.cpp", - "lib/Conversion/ComplexToStandard/*.h", ]), hdrs = glob([ "include/mlir/Conversion/ComplexToStandard/*.h", @@ -12760,7 +12627,6 @@ cc_library( name = "ArithTransforms", srcs = glob([ "lib/Dialect/Arith/Transforms/*.cpp", - "lib/Dialect/Arith/Transforms/*.h", ]), hdrs = glob([ "include/mlir/Dialect/Arith/Transforms/*.h", @@ -12882,7 +12748,6 @@ cc_library( srcs = glob( [ "lib/Dialect/Math/IR/*.cpp", - "lib/Dialect/Math/IR/*.h", ], ), hdrs = [ @@ -12909,7 +12774,6 @@ cc_library( name = "MathTransforms", srcs = glob([ "lib/Dialect/Math/Transforms/*.cpp", - "lib/Dialect/Math/Transforms/*.h", ]), hdrs = glob(["include/mlir/Dialect/Math/Transforms/*.h"]), includes = ["include"], @@ -12933,7 +12797,6 @@ cc_library( name = "MathToLibm", srcs = glob([ "lib/Conversion/MathToLibm/*.cpp", - "lib/Conversion/MathToLibm/*.h", ]), hdrs = glob([ "include/mlir/Conversion/MathToLibm/*.h", @@ -13024,7 +12887,6 @@ cc_library( srcs = glob( [ "lib/Dialect/MemRef/IR/*.cpp", - "lib/Dialect/MemRef/IR/*.h", ], ), hdrs = [ @@ -13102,7 +12964,6 @@ cc_library( srcs = glob( [ "lib/Dialect/MemRef/Transforms/*.cpp", - "lib/Dialect/MemRef/Transforms/*.h", ], ), hdrs = glob(["include/mlir/Dialect/MemRef/Transforms/*.h"]), @@ -13730,7 +13591,6 @@ cc_library( srcs = glob( [ "lib/Dialect/Bufferization/Transforms/*.cpp", - "lib/Dialect/Bufferization/Transforms/*.h", ], ), hdrs = glob(["include/mlir/Dialect/Bufferization/Transforms/*.h"]), @@ -13922,7 +13782,6 @@ cc_library( srcs = glob( [ "lib/Tools/PDLL/ODS/*.cpp", - "lib/Tools/PDLL/ODS/*.h", ], ), hdrs = glob(["include/mlir/Tools/PDLL/ODS/*.h"]), @@ -13956,7 +13815,6 @@ cc_library( srcs = glob( [ "lib/Tools/PDLL/CodeGen/*.cpp", - "lib/Tools/PDLL/CodeGen/*.h", ], ), hdrs = glob(["include/mlir/Tools/PDLL/CodeGen/*.h"]), @@ -14117,7 +13975,6 @@ cc_library( name = "UBToLLVM", srcs = glob([ "lib/Conversion/UBToLLVM/*.cpp", - "lib/Conversion/UBToLLVM/*.h", ]), hdrs = glob([ "include/mlir/Conversion/UBToLLVM/*.h", @@ -14138,7 +13995,6 @@ cc_library( name = "UBToSPIRV", srcs = glob([ "lib/Conversion/UBToSPIRV/*.cpp", - "lib/Conversion/UBToSPIRV/*.h", ]), hdrs = glob([ "include/mlir/Conversion/UBToSPIRV/*.h", diff --git a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel index df2392fd8c6e..57b29eb46e08 100644 --- a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel @@ -818,7 +818,6 @@ cc_library( cc_library( name = "TestMesh", srcs = glob(["lib/Dialect/Mesh/**/*.cpp"]), - hdrs = glob(["lib/Dialect/Mesh/**/*.h"]), includes = ["lib/Dialect/Test"], deps = [ ":TestDialect", @@ -1046,18 +1045,6 @@ cc_library( ], ) -cc_library( - name = "TestTypeDialect", - srcs = glob([ - "lib/Dialect/LLVMIR/*.cpp", - ]), - deps = [ - ":TestDialect", - "//mlir:IR", - "//mlir:LLVMDialect", - ], -) - cc_library( name = "TestTosaDialect", srcs = glob([ diff --git a/utils/bazel/llvm-project-overlay/mlir/unittests/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/unittests/BUILD.bazel index c6b630230bb3..7172beb4de9a 100644 --- a/utils/bazel/llvm-project-overlay/mlir/unittests/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/unittests/BUILD.bazel @@ -16,7 +16,6 @@ cc_test( size = "small", srcs = glob([ "Debug/*.cpp", - "Debug/*.h", ]), deps = [ "//llvm:Support", @@ -35,7 +34,6 @@ cc_test( size = "small", srcs = glob([ "IR/*.cpp", - "IR/*.h", ]), deps = [ "//llvm:Support", @@ -57,7 +55,6 @@ cc_test( size = "small", srcs = glob([ "Interfaces/*.cpp", - "Interfaces/*.h", ]), deps = [ "//llvm:Support", @@ -82,7 +79,6 @@ cc_test( size = "small", srcs = glob([ "Support/*.cpp", - "Support/*.h", ]), deps = [ "//llvm:Support", @@ -98,7 +94,6 @@ cc_test( size = "small", srcs = glob([ "Pass/*.cpp", - "Pass/*.h", ]), deps = [ "//llvm:Support", @@ -118,7 +113,6 @@ cc_test( size = "small", srcs = glob([ "Rewrite/*.cpp", - "Rewrite/*.h", ]), deps = [ "//mlir:IR", @@ -134,7 +128,6 @@ cc_test( size = "small", srcs = glob([ "Dialect/*.cpp", - "Dialect/*.h", ]), deps = [ "//llvm:Support", @@ -149,7 +142,6 @@ cc_test( size = "small", srcs = glob([ "Dialect/MemRef/*.cpp", - "Dialect/MemRef/*.h", ]), deps = [ "//llvm:TestingSupport", @@ -161,27 +153,11 @@ cc_test( ], ) -cc_test( - name = "quantops_tests", - size = "small", - srcs = glob([ - "Dialect/Quant/*.cpp", - "Dialect/Quant/*.h", - ]), - deps = [ - "//llvm:TestingSupport", - "//mlir:QuantOps", - "//mlir:Transforms", - "//third-party/unittest:gtest_main", - ], -) - cc_test( name = "scf_tests", size = "small", srcs = glob([ "Dialect/SCF/*.cpp", - "Dialect/SCF/*.h", ]), deps = [ "//mlir:ArithDialect", @@ -200,7 +176,6 @@ cc_test( size = "small", srcs = glob([ "Dialect/SparseTensor/*.cpp", - "Dialect/SparseTensor/*.h", ]), deps = [ "//llvm:Support", @@ -218,7 +193,6 @@ cc_test( size = "small", srcs = glob([ "Dialect/SPIRV/*.cpp", - "Dialect/SPIRV/*.h", ]), deps = [ "//llvm:Support", @@ -238,7 +212,6 @@ cc_test( size = "small", srcs = glob([ "Dialect/Transform/*.cpp", - "Dialect/Transform/*.h", ]), deps = [ "//llvm:Support", @@ -264,7 +237,6 @@ cc_test( size = "small", srcs = glob([ "Dialect/Utils/*.cpp", - "Dialect/Utils/*.h", ]), deps = [ "//llvm:Support", @@ -317,7 +289,6 @@ cc_test( size = "small", srcs = glob([ "TableGen/*.cpp", - "TableGen/*.h", ]) + [ "TableGen/EnumsGenTest.cpp.inc", "TableGen/EnumsGenTest.h.inc", @@ -343,7 +314,6 @@ cc_test( size = "small", srcs = glob([ "Transforms/*.cpp", - "Transforms/*.h", ]), deps = [ "//mlir:AffineAnalysis", @@ -363,8 +333,6 @@ cc_test( name = "analysis_tests", size = "small", srcs = glob([ - "Analysis/*.cpp", - "Analysis/*.h", "Analysis/*/*.cpp", "Analysis/*/*.h", ]), @@ -387,9 +355,6 @@ cc_test( size = "small", srcs = glob([ "Bytecode/*.cpp", - "Bytecode/*.h", - "Bytecode/*/*.cpp", - "Bytecode/*/*.h", ]), deps = [ "//llvm:Support", @@ -408,10 +373,7 @@ cc_test( name = "conversion_tests", size = "small", srcs = glob([ - "Conversion/*.cpp", - "Conversion/*.h", "Conversion/*/*.cpp", - "Conversion/*/*.h", ]), deps = [ "//mlir:ArithDialect", -- GitLab From 5184e6ad69b0ca69dfba6fb0982a675c595f49a2 Mon Sep 17 00:00:00 2001 From: Aaron Ballman Date: Fri, 22 Mar 2024 13:00:32 -0400 Subject: [PATCH 082/306] Removing accidental code from 527a624205748814dd9309eda7ee308b40b2359a --- clang/test/C/C11/n1365.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clang/test/C/C11/n1365.c b/clang/test/C/C11/n1365.c index 3f769faa365d..d60bb546b29a 100644 --- a/clang/test/C/C11/n1365.c +++ b/clang/test/C/C11/n1365.c @@ -23,7 +23,7 @@ void func(void) { // CHECK-NEXT: FloatingLiteral // Ensure that a cast removes the extra precision. - _Static_assert((float)(123.0F * 2.0F) == 246.0F, ""); + _Static_assert(123.0F * 2.0F == 246.0F, ""); // CHECK: StaticAssertDecl // CHECK-NEXT: ImplicitCastExpr {{.*}} '_Bool' // CHECK-NEXT: BinaryOperator {{.*}} 'int' '==' -- GitLab From bd493756fa51e538575fc320aae50d75394f0567 Mon Sep 17 00:00:00 2001 From: Aiden Grossman Date: Fri, 22 Mar 2024 17:01:39 +0000 Subject: [PATCH 083/306] [llvm-exegesis] Refactor parent code to separate function (#86232) This patch refactors the parent code to a separate function in the subprocess executor to make the code more clear and easy to follow. --- .../llvm-exegesis/lib/BenchmarkRunner.cpp | 116 ++++++++++-------- 1 file changed, 63 insertions(+), 53 deletions(-) diff --git a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp index f0452605eb24..6d8da7bd86e3 100644 --- a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp +++ b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp @@ -278,59 +278,20 @@ private: return FD; } - Error createSubProcessAndRunBenchmark( - StringRef CounterName, SmallVectorImpl &CounterValues, - ArrayRef ValidationCounters, - SmallVectorImpl &ValidationCounterValues) const { - int PipeFiles[2]; - int PipeSuccessOrErr = socketpair(AF_UNIX, SOCK_DGRAM, 0, PipeFiles); - if (PipeSuccessOrErr != 0) { - return make_error( - "Failed to create a pipe for interprocess communication between " - "llvm-exegesis and the benchmarking subprocess: " + - Twine(strerror(errno))); - } - - SubprocessMemory SPMemory; - Error MemoryInitError = SPMemory.initializeSubprocessMemory(getpid()); - if (MemoryInitError) - return MemoryInitError; - - Error AddMemDefError = - SPMemory.addMemoryDefinition(Key.MemoryValues, getpid()); - if (AddMemDefError) - return AddMemDefError; - - pid_t ParentOrChildPID = fork(); - - if (ParentOrChildPID == -1) { - return make_error("Failed to create child process: " + - Twine(strerror(errno))); - } - - if (ParentOrChildPID == 0) { - // We are in the child process, close the write end of the pipe. - close(PipeFiles[1]); - // Unregister handlers, signal handling is now handled through ptrace in - // the host process. - sys::unregisterHandlers(); - prepareAndRunBenchmark(PipeFiles[0], Key); - // The child process terminates in the above function, so we should never - // get to this point. - llvm_unreachable("Child process didn't exit when expected."); - } - + Error + runParentProcess(pid_t ChildPID, int WriteFD, StringRef CounterName, + SmallVectorImpl &CounterValues, + ArrayRef ValidationCounters, + SmallVectorImpl &ValidationCounterValues) const { const ExegesisTarget &ET = State.getExegesisTarget(); - auto CounterOrError = ET.createCounter( - CounterName, State, ValidationCounters, ParentOrChildPID); + auto CounterOrError = + ET.createCounter(CounterName, State, ValidationCounters, ChildPID); if (!CounterOrError) return CounterOrError.takeError(); pfm::CounterGroup *Counter = CounterOrError.get().get(); - close(PipeFiles[0]); - // Make sure to attach to the process (and wait for the sigstop to be // delivered and for the process to continue) before we write to the counter // file descriptor. Attaching to the process before writing to the socket @@ -338,7 +299,7 @@ private: // attach afterwards, the subprocess might exit before we get to the attach // call due to effects like scheduler contention, introducing transient // failures. - if (ptrace(PTRACE_ATTACH, ParentOrChildPID, NULL, NULL) != 0) + if (ptrace(PTRACE_ATTACH, ChildPID, NULL, NULL) != 0) return make_error("Failed to attach to the child process: " + Twine(strerror(errno))); @@ -348,14 +309,14 @@ private: Twine(strerror(errno))); } - if (ptrace(PTRACE_CONT, ParentOrChildPID, NULL, NULL) != 0) + if (ptrace(PTRACE_CONT, ChildPID, NULL, NULL) != 0) return make_error( "Failed to continue execution of the child process: " + Twine(strerror(errno))); int CounterFileDescriptor = Counter->getFileDescriptor(); Error SendError = - sendFileDescriptorThroughSocket(PipeFiles[1], CounterFileDescriptor); + sendFileDescriptorThroughSocket(WriteFD, CounterFileDescriptor); if (SendError) return SendError; @@ -395,8 +356,7 @@ private: // An error was encountered running the snippet, process it siginfo_t ChildSignalInfo; - if (ptrace(PTRACE_GETSIGINFO, ParentOrChildPID, NULL, &ChildSignalInfo) == - -1) { + if (ptrace(PTRACE_GETSIGINFO, ChildPID, NULL, &ChildSignalInfo) == -1) { return make_error("Getting signal info from the child failed: " + Twine(strerror(errno))); } @@ -422,6 +382,56 @@ private: return make_error(ChildSignalInfo.si_signo); } + Error createSubProcessAndRunBenchmark( + StringRef CounterName, SmallVectorImpl &CounterValues, + ArrayRef ValidationCounters, + SmallVectorImpl &ValidationCounterValues) const { + int PipeFiles[2]; + int PipeSuccessOrErr = socketpair(AF_UNIX, SOCK_DGRAM, 0, PipeFiles); + if (PipeSuccessOrErr != 0) { + return make_error( + "Failed to create a pipe for interprocess communication between " + "llvm-exegesis and the benchmarking subprocess: " + + Twine(strerror(errno))); + } + + SubprocessMemory SPMemory; + Error MemoryInitError = SPMemory.initializeSubprocessMemory(getpid()); + if (MemoryInitError) + return MemoryInitError; + + Error AddMemDefError = + SPMemory.addMemoryDefinition(Key.MemoryValues, getpid()); + if (AddMemDefError) + return AddMemDefError; + + pid_t ParentOrChildPID = fork(); + + if (ParentOrChildPID == -1) { + return make_error("Failed to create child process: " + + Twine(strerror(errno))); + } + + if (ParentOrChildPID == 0) { + // We are in the child process, close the write end of the pipe. + close(PipeFiles[1]); + // Unregister handlers, signal handling is now handled through ptrace in + // the host process. + sys::unregisterHandlers(); + runChildSubprocess(PipeFiles[0], Key); + // The child process terminates in the above function, so we should never + // get to this point. + llvm_unreachable("Child process didn't exit when expected."); + } + + // Close the read end of the pipe as we only need to write to the subprocess + // from the parent process. + close(PipeFiles[0]); + return runParentProcess(ParentOrChildPID, PipeFiles[1], CounterName, + CounterValues, ValidationCounters, + ValidationCounterValues); + } + void disableCoreDumps() const { struct rlimit rlim; @@ -429,8 +439,8 @@ private: setrlimit(RLIMIT_CORE, &rlim); } - [[noreturn]] void prepareAndRunBenchmark(int Pipe, - const BenchmarkKey &Key) const { + [[noreturn]] void runChildSubprocess(int Pipe, + const BenchmarkKey &Key) const { // Disable core dumps in the child process as otherwise everytime we // encounter an execution failure like a segmentation fault, we will create // a core dump. We report the information directly rather than require the -- GitLab From d2f8ba7d6dc7251815f1431cf8715053576615f4 Mon Sep 17 00:00:00 2001 From: Sacha Coppey Date: Fri, 22 Mar 2024 18:08:13 +0100 Subject: [PATCH 084/306] [RISCV][NFC] Add generateMCInstSeq in RISCVMatInt (#84462) This allows to avoid duplicating the code handling the instructions outputted by `generateInstSeq` when emitting `MCInst`s. --- .../Target/RISCV/AsmParser/RISCVAsmParser.cpp | 31 ++------------- .../Target/RISCV/MCTargetDesc/RISCVMatInt.cpp | 38 +++++++++++++++++++ .../Target/RISCV/MCTargetDesc/RISCVMatInt.h | 5 +++ 3 files changed, 47 insertions(+), 27 deletions(-) diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp index 1779959324da..cb2ba52390e2 100644 --- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp +++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp @@ -3081,34 +3081,11 @@ void RISCVAsmParser::emitToStreamer(MCStreamer &S, const MCInst &Inst) { void RISCVAsmParser::emitLoadImm(MCRegister DestReg, int64_t Value, MCStreamer &Out) { - RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Value, getSTI()); - - MCRegister SrcReg = RISCV::X0; - for (const RISCVMatInt::Inst &Inst : Seq) { - switch (Inst.getOpndKind()) { - case RISCVMatInt::Imm: - emitToStreamer(Out, - MCInstBuilder(Inst.getOpcode()).addReg(DestReg).addImm(Inst.getImm())); - break; - case RISCVMatInt::RegX0: - emitToStreamer( - Out, MCInstBuilder(Inst.getOpcode()).addReg(DestReg).addReg(SrcReg).addReg( - RISCV::X0)); - break; - case RISCVMatInt::RegReg: - emitToStreamer( - Out, MCInstBuilder(Inst.getOpcode()).addReg(DestReg).addReg(SrcReg).addReg( - SrcReg)); - break; - case RISCVMatInt::RegImm: - emitToStreamer( - Out, MCInstBuilder(Inst.getOpcode()).addReg(DestReg).addReg(SrcReg).addImm( - Inst.getImm())); - break; - } + SmallVector Seq; + RISCVMatInt::generateMCInstSeq(Value, getSTI(), DestReg, Seq); - // Only the first instruction has X0 as its source. - SrcReg = DestReg; + for (MCInst &Inst : Seq) { + emitToStreamer(Out, Inst); } } diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.cpp index 4358a5b878e6..c3bae152993e 100644 --- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.cpp +++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.cpp @@ -9,6 +9,7 @@ #include "RISCVMatInt.h" #include "MCTargetDesc/RISCVMCTargetDesc.h" #include "llvm/ADT/APInt.h" +#include "llvm/MC/MCInstBuilder.h" #include "llvm/Support/MathExtras.h" using namespace llvm; @@ -436,6 +437,43 @@ InstSeq generateInstSeq(int64_t Val, const MCSubtargetInfo &STI) { return Res; } +void generateMCInstSeq(int64_t Val, const MCSubtargetInfo &STI, + MCRegister DestReg, SmallVectorImpl &Insts) { + RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val, STI); + + MCRegister SrcReg = RISCV::X0; + for (RISCVMatInt::Inst &Inst : Seq) { + switch (Inst.getOpndKind()) { + case RISCVMatInt::Imm: + Insts.push_back(MCInstBuilder(Inst.getOpcode()) + .addReg(DestReg) + .addImm(Inst.getImm())); + break; + case RISCVMatInt::RegX0: + Insts.push_back(MCInstBuilder(Inst.getOpcode()) + .addReg(DestReg) + .addReg(SrcReg) + .addReg(RISCV::X0)); + break; + case RISCVMatInt::RegReg: + Insts.push_back(MCInstBuilder(Inst.getOpcode()) + .addReg(DestReg) + .addReg(SrcReg) + .addReg(SrcReg)); + break; + case RISCVMatInt::RegImm: + Insts.push_back(MCInstBuilder(Inst.getOpcode()) + .addReg(DestReg) + .addReg(SrcReg) + .addImm(Inst.getImm())); + break; + } + + // Only the first instruction has X0 as its source. + SrcReg = DestReg; + } +} + InstSeq generateTwoRegInstSeq(int64_t Val, const MCSubtargetInfo &STI, unsigned &ShiftAmt, unsigned &AddOpc) { int64_t LoVal = SignExtend64<32>(Val); diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.h index 780f685463f3..e87e0f325647 100644 --- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.h +++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.h @@ -10,6 +10,7 @@ #define LLVM_LIB_TARGET_RISCV_MCTARGETDESC_MATINT_H #include "llvm/ADT/SmallVector.h" +#include "llvm/MC/MCRegister.h" #include "llvm/MC/MCSubtargetInfo.h" #include @@ -48,6 +49,10 @@ using InstSeq = SmallVector; // instruction selection. InstSeq generateInstSeq(int64_t Val, const MCSubtargetInfo &STI); +// Helper to generate the generateInstSeq instruction sequence using MCInsts +void generateMCInstSeq(int64_t Val, const MCSubtargetInfo &STI, + MCRegister DestReg, SmallVectorImpl &Insts); + // Helper to generate an instruction sequence that can materialize the given // immediate value into a register using an additional temporary register. This // handles cases where the constant can be generated by (ADD (SLLI X, C), X) or -- GitLab From c3a41aac5f32475b9a0499e6e888e713763566dc Mon Sep 17 00:00:00 2001 From: Aiden Grossman Date: Fri, 22 Mar 2024 10:25:23 -0700 Subject: [PATCH 085/306] Revert "[llvm-exegesis] Refactor parent code to separate function (#86232)" This reverts commit bd493756fa51e538575fc320aae50d75394f0567. Causes build failures on non-X86 platforms. https://lab.llvm.org/buildbot/#/changes/128363 --- .../llvm-exegesis/lib/BenchmarkRunner.cpp | 116 ++++++++---------- 1 file changed, 53 insertions(+), 63 deletions(-) diff --git a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp index 6d8da7bd86e3..f0452605eb24 100644 --- a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp +++ b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp @@ -278,20 +278,59 @@ private: return FD; } - Error - runParentProcess(pid_t ChildPID, int WriteFD, StringRef CounterName, - SmallVectorImpl &CounterValues, - ArrayRef ValidationCounters, - SmallVectorImpl &ValidationCounterValues) const { + Error createSubProcessAndRunBenchmark( + StringRef CounterName, SmallVectorImpl &CounterValues, + ArrayRef ValidationCounters, + SmallVectorImpl &ValidationCounterValues) const { + int PipeFiles[2]; + int PipeSuccessOrErr = socketpair(AF_UNIX, SOCK_DGRAM, 0, PipeFiles); + if (PipeSuccessOrErr != 0) { + return make_error( + "Failed to create a pipe for interprocess communication between " + "llvm-exegesis and the benchmarking subprocess: " + + Twine(strerror(errno))); + } + + SubprocessMemory SPMemory; + Error MemoryInitError = SPMemory.initializeSubprocessMemory(getpid()); + if (MemoryInitError) + return MemoryInitError; + + Error AddMemDefError = + SPMemory.addMemoryDefinition(Key.MemoryValues, getpid()); + if (AddMemDefError) + return AddMemDefError; + + pid_t ParentOrChildPID = fork(); + + if (ParentOrChildPID == -1) { + return make_error("Failed to create child process: " + + Twine(strerror(errno))); + } + + if (ParentOrChildPID == 0) { + // We are in the child process, close the write end of the pipe. + close(PipeFiles[1]); + // Unregister handlers, signal handling is now handled through ptrace in + // the host process. + sys::unregisterHandlers(); + prepareAndRunBenchmark(PipeFiles[0], Key); + // The child process terminates in the above function, so we should never + // get to this point. + llvm_unreachable("Child process didn't exit when expected."); + } + const ExegesisTarget &ET = State.getExegesisTarget(); - auto CounterOrError = - ET.createCounter(CounterName, State, ValidationCounters, ChildPID); + auto CounterOrError = ET.createCounter( + CounterName, State, ValidationCounters, ParentOrChildPID); if (!CounterOrError) return CounterOrError.takeError(); pfm::CounterGroup *Counter = CounterOrError.get().get(); + close(PipeFiles[0]); + // Make sure to attach to the process (and wait for the sigstop to be // delivered and for the process to continue) before we write to the counter // file descriptor. Attaching to the process before writing to the socket @@ -299,7 +338,7 @@ private: // attach afterwards, the subprocess might exit before we get to the attach // call due to effects like scheduler contention, introducing transient // failures. - if (ptrace(PTRACE_ATTACH, ChildPID, NULL, NULL) != 0) + if (ptrace(PTRACE_ATTACH, ParentOrChildPID, NULL, NULL) != 0) return make_error("Failed to attach to the child process: " + Twine(strerror(errno))); @@ -309,14 +348,14 @@ private: Twine(strerror(errno))); } - if (ptrace(PTRACE_CONT, ChildPID, NULL, NULL) != 0) + if (ptrace(PTRACE_CONT, ParentOrChildPID, NULL, NULL) != 0) return make_error( "Failed to continue execution of the child process: " + Twine(strerror(errno))); int CounterFileDescriptor = Counter->getFileDescriptor(); Error SendError = - sendFileDescriptorThroughSocket(WriteFD, CounterFileDescriptor); + sendFileDescriptorThroughSocket(PipeFiles[1], CounterFileDescriptor); if (SendError) return SendError; @@ -356,7 +395,8 @@ private: // An error was encountered running the snippet, process it siginfo_t ChildSignalInfo; - if (ptrace(PTRACE_GETSIGINFO, ChildPID, NULL, &ChildSignalInfo) == -1) { + if (ptrace(PTRACE_GETSIGINFO, ParentOrChildPID, NULL, &ChildSignalInfo) == + -1) { return make_error("Getting signal info from the child failed: " + Twine(strerror(errno))); } @@ -382,56 +422,6 @@ private: return make_error(ChildSignalInfo.si_signo); } - Error createSubProcessAndRunBenchmark( - StringRef CounterName, SmallVectorImpl &CounterValues, - ArrayRef ValidationCounters, - SmallVectorImpl &ValidationCounterValues) const { - int PipeFiles[2]; - int PipeSuccessOrErr = socketpair(AF_UNIX, SOCK_DGRAM, 0, PipeFiles); - if (PipeSuccessOrErr != 0) { - return make_error( - "Failed to create a pipe for interprocess communication between " - "llvm-exegesis and the benchmarking subprocess: " + - Twine(strerror(errno))); - } - - SubprocessMemory SPMemory; - Error MemoryInitError = SPMemory.initializeSubprocessMemory(getpid()); - if (MemoryInitError) - return MemoryInitError; - - Error AddMemDefError = - SPMemory.addMemoryDefinition(Key.MemoryValues, getpid()); - if (AddMemDefError) - return AddMemDefError; - - pid_t ParentOrChildPID = fork(); - - if (ParentOrChildPID == -1) { - return make_error("Failed to create child process: " + - Twine(strerror(errno))); - } - - if (ParentOrChildPID == 0) { - // We are in the child process, close the write end of the pipe. - close(PipeFiles[1]); - // Unregister handlers, signal handling is now handled through ptrace in - // the host process. - sys::unregisterHandlers(); - runChildSubprocess(PipeFiles[0], Key); - // The child process terminates in the above function, so we should never - // get to this point. - llvm_unreachable("Child process didn't exit when expected."); - } - - // Close the read end of the pipe as we only need to write to the subprocess - // from the parent process. - close(PipeFiles[0]); - return runParentProcess(ParentOrChildPID, PipeFiles[1], CounterName, - CounterValues, ValidationCounters, - ValidationCounterValues); - } - void disableCoreDumps() const { struct rlimit rlim; @@ -439,8 +429,8 @@ private: setrlimit(RLIMIT_CORE, &rlim); } - [[noreturn]] void runChildSubprocess(int Pipe, - const BenchmarkKey &Key) const { + [[noreturn]] void prepareAndRunBenchmark(int Pipe, + const BenchmarkKey &Key) const { // Disable core dumps in the child process as otherwise everytime we // encounter an execution failure like a segmentation fault, we will create // a core dump. We report the information directly rather than require the -- GitLab From 36a6afdd2c7fa02548260ebe4c993b705c6e6e38 Mon Sep 17 00:00:00 2001 From: Aiden Grossman Date: Fri, 22 Mar 2024 10:26:24 -0700 Subject: [PATCH 086/306] Reland "[llvm-exegesis] Refactor parent code to separate function (#86232)" This reverts commit c3a41aac5f32475b9a0499e6e888e713763566dc. This relands commit bd493756fa51e538575fc320aae50d75394f0567. Apparently I forgot to update a couple values, so this change failed on every builder that builds those sections (should be every Linux platform) rather than something architecture specific like originally thought. I swore I updated the values and ran check-llvm before merging. Wondering If I forgot to push those changes... --- .../llvm-exegesis/lib/BenchmarkRunner.cpp | 124 ++++++++++-------- 1 file changed, 67 insertions(+), 57 deletions(-) diff --git a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp index f0452605eb24..6d5f0286fb9c 100644 --- a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp +++ b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp @@ -278,59 +278,20 @@ private: return FD; } - Error createSubProcessAndRunBenchmark( - StringRef CounterName, SmallVectorImpl &CounterValues, - ArrayRef ValidationCounters, - SmallVectorImpl &ValidationCounterValues) const { - int PipeFiles[2]; - int PipeSuccessOrErr = socketpair(AF_UNIX, SOCK_DGRAM, 0, PipeFiles); - if (PipeSuccessOrErr != 0) { - return make_error( - "Failed to create a pipe for interprocess communication between " - "llvm-exegesis and the benchmarking subprocess: " + - Twine(strerror(errno))); - } - - SubprocessMemory SPMemory; - Error MemoryInitError = SPMemory.initializeSubprocessMemory(getpid()); - if (MemoryInitError) - return MemoryInitError; - - Error AddMemDefError = - SPMemory.addMemoryDefinition(Key.MemoryValues, getpid()); - if (AddMemDefError) - return AddMemDefError; - - pid_t ParentOrChildPID = fork(); - - if (ParentOrChildPID == -1) { - return make_error("Failed to create child process: " + - Twine(strerror(errno))); - } - - if (ParentOrChildPID == 0) { - // We are in the child process, close the write end of the pipe. - close(PipeFiles[1]); - // Unregister handlers, signal handling is now handled through ptrace in - // the host process. - sys::unregisterHandlers(); - prepareAndRunBenchmark(PipeFiles[0], Key); - // The child process terminates in the above function, so we should never - // get to this point. - llvm_unreachable("Child process didn't exit when expected."); - } - + Error + runParentProcess(pid_t ChildPID, int WriteFD, StringRef CounterName, + SmallVectorImpl &CounterValues, + ArrayRef ValidationCounters, + SmallVectorImpl &ValidationCounterValues) const { const ExegesisTarget &ET = State.getExegesisTarget(); - auto CounterOrError = ET.createCounter( - CounterName, State, ValidationCounters, ParentOrChildPID); + auto CounterOrError = + ET.createCounter(CounterName, State, ValidationCounters, ChildPID); if (!CounterOrError) return CounterOrError.takeError(); pfm::CounterGroup *Counter = CounterOrError.get().get(); - close(PipeFiles[0]); - // Make sure to attach to the process (and wait for the sigstop to be // delivered and for the process to continue) before we write to the counter // file descriptor. Attaching to the process before writing to the socket @@ -338,30 +299,30 @@ private: // attach afterwards, the subprocess might exit before we get to the attach // call due to effects like scheduler contention, introducing transient // failures. - if (ptrace(PTRACE_ATTACH, ParentOrChildPID, NULL, NULL) != 0) + if (ptrace(PTRACE_ATTACH, ChildPID, NULL, NULL) != 0) return make_error("Failed to attach to the child process: " + Twine(strerror(errno))); - if (waitpid(ParentOrChildPID, NULL, 0) == -1) { + if (waitpid(ChildPID, NULL, 0) == -1) { return make_error( "Failed to wait for child process to stop after attaching: " + Twine(strerror(errno))); } - if (ptrace(PTRACE_CONT, ParentOrChildPID, NULL, NULL) != 0) + if (ptrace(PTRACE_CONT, ChildPID, NULL, NULL) != 0) return make_error( "Failed to continue execution of the child process: " + Twine(strerror(errno))); int CounterFileDescriptor = Counter->getFileDescriptor(); Error SendError = - sendFileDescriptorThroughSocket(PipeFiles[1], CounterFileDescriptor); + sendFileDescriptorThroughSocket(WriteFD, CounterFileDescriptor); if (SendError) return SendError; int ChildStatus; - if (waitpid(ParentOrChildPID, &ChildStatus, 0) == -1) { + if (waitpid(ChildPID, &ChildStatus, 0) == -1) { return make_error( "Waiting for the child process to complete failed: " + Twine(strerror(errno))); @@ -395,8 +356,7 @@ private: // An error was encountered running the snippet, process it siginfo_t ChildSignalInfo; - if (ptrace(PTRACE_GETSIGINFO, ParentOrChildPID, NULL, &ChildSignalInfo) == - -1) { + if (ptrace(PTRACE_GETSIGINFO, ChildPID, NULL, &ChildSignalInfo) == -1) { return make_error("Getting signal info from the child failed: " + Twine(strerror(errno))); } @@ -405,13 +365,13 @@ private: // handlers to run, and calling SIGTERM would mean that ptrace will force // it to block in the signal-delivery-stop for the SIGSEGV/other signals, // and upon exit. - if (kill(ParentOrChildPID, SIGKILL) == -1) + if (kill(ChildPID, SIGKILL) == -1) return make_error("Failed to kill child benchmarking proces: " + Twine(strerror(errno))); // Wait for the process to exit so that there are no zombie processes left // around. - if (waitpid(ParentOrChildPID, NULL, 0) == -1) + if (waitpid(ChildPID, NULL, 0) == -1) return make_error("Failed to wait for process to die: " + Twine(strerror(errno))); @@ -422,6 +382,56 @@ private: return make_error(ChildSignalInfo.si_signo); } + Error createSubProcessAndRunBenchmark( + StringRef CounterName, SmallVectorImpl &CounterValues, + ArrayRef ValidationCounters, + SmallVectorImpl &ValidationCounterValues) const { + int PipeFiles[2]; + int PipeSuccessOrErr = socketpair(AF_UNIX, SOCK_DGRAM, 0, PipeFiles); + if (PipeSuccessOrErr != 0) { + return make_error( + "Failed to create a pipe for interprocess communication between " + "llvm-exegesis and the benchmarking subprocess: " + + Twine(strerror(errno))); + } + + SubprocessMemory SPMemory; + Error MemoryInitError = SPMemory.initializeSubprocessMemory(getpid()); + if (MemoryInitError) + return MemoryInitError; + + Error AddMemDefError = + SPMemory.addMemoryDefinition(Key.MemoryValues, getpid()); + if (AddMemDefError) + return AddMemDefError; + + pid_t ParentOrChildPID = fork(); + + if (ParentOrChildPID == -1) { + return make_error("Failed to create child process: " + + Twine(strerror(errno))); + } + + if (ParentOrChildPID == 0) { + // We are in the child process, close the write end of the pipe. + close(PipeFiles[1]); + // Unregister handlers, signal handling is now handled through ptrace in + // the host process. + sys::unregisterHandlers(); + runChildSubprocess(PipeFiles[0], Key); + // The child process terminates in the above function, so we should never + // get to this point. + llvm_unreachable("Child process didn't exit when expected."); + } + + // Close the read end of the pipe as we only need to write to the subprocess + // from the parent process. + close(PipeFiles[0]); + return runParentProcess(ParentOrChildPID, PipeFiles[1], CounterName, + CounterValues, ValidationCounters, + ValidationCounterValues); + } + void disableCoreDumps() const { struct rlimit rlim; @@ -429,8 +439,8 @@ private: setrlimit(RLIMIT_CORE, &rlim); } - [[noreturn]] void prepareAndRunBenchmark(int Pipe, - const BenchmarkKey &Key) const { + [[noreturn]] void runChildSubprocess(int Pipe, + const BenchmarkKey &Key) const { // Disable core dumps in the child process as otherwise everytime we // encounter an execution failure like a segmentation fault, we will create // a core dump. We report the information directly rather than require the -- GitLab From bbcfe6f4311af8cf6095a5bc5937fa68a87b4289 Mon Sep 17 00:00:00 2001 From: srcarroll <50210727+srcarroll@users.noreply.github.com> Date: Fri, 22 Mar 2024 12:37:39 -0500 Subject: [PATCH 087/306] [mlir][transform] Emit error message with `emitSilenceableFailure` (#86146) The previous implementation used a `notifyMatchFailure` to emit failure message inappropriately and then used the `emitDefaultSilenceableFailure`. This patch changes this to use the more appropriate `emitSilenceableFailure` with error message. Additionally a failure test has been added. --- .../TransformOps/LinalgTransformOps.cpp | 14 ++++---- .../Dialect/Linalg/flatten-unsupported.mlir | 33 +++++++++++++++++++ 2 files changed, 41 insertions(+), 6 deletions(-) create mode 100644 mlir/test/Dialect/Linalg/flatten-unsupported.mlir diff --git a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp index ecf998312482..88819cd96435 100644 --- a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp +++ b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp @@ -3269,22 +3269,24 @@ DiagnosedSilenceableFailure transform::FlattenElementwiseLinalgOp::applyToOne( transform::ApplyToEachResultList &results, transform::TransformState &state) { rewriter.setInsertionPoint(target); - if (!isElementwise(target)) { - failed(rewriter.notifyMatchFailure( - target, "only elementwise flattening is supported")); - return emitDefaultSilenceableFailure(target); - } + if (!isElementwise(target)) + return mlir::emitSilenceableFailure(target->getLoc()) + << "only elementwise flattening is supported"; + // If rank <= 1, do nothing if (target.getNumLoops() <= 1) { results.push_back(target); return DiagnosedSilenceableFailure::success(); } + + // Attempt to flatten all dims to one. ReassociationIndices reassociation(target.getNumLoops()); std::iota(reassociation.begin(), reassociation.end(), 0); auto maybeFlattened = collapseOpIterationDims(target, reassociation, rewriter); if (failed(maybeFlattened)) - return emitDefaultSilenceableFailure(target); + return mlir::emitSilenceableFailure(target->getLoc()) + << "attempted to flatten, but failed"; results.push_back(maybeFlattened->collapsedOp); rewriter.replaceOp(target, maybeFlattened->results); return DiagnosedSilenceableFailure::success(); diff --git a/mlir/test/Dialect/Linalg/flatten-unsupported.mlir b/mlir/test/Dialect/Linalg/flatten-unsupported.mlir new file mode 100644 index 000000000000..499db4cfb329 --- /dev/null +++ b/mlir/test/Dialect/Linalg/flatten-unsupported.mlir @@ -0,0 +1,33 @@ +// RUN: mlir-opt %s -transform-interpreter -split-input-file -verify-diagnostics + +func.func @non_elementwise(%arg0: memref<2x3xf32>, %arg1: memref<3x4xf32>, %arg2: memref<2x4xf32>) { + // expected-error @below {{only elementwise flattening is supported}} + linalg.matmul ins(%arg0, %arg1 : memref<2x3xf32>, memref<3x4xf32>) outs(%arg2: memref<2x4xf32>) + return +} + +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) { + %0 = transform.structured.match interface{LinalgOp} in %arg1 : (!transform.any_op) -> !transform.any_op + %flattened = transform.structured.flatten_elementwise %0 + : (!transform.any_op) -> !transform.any_op + transform.yield + } +} + +// ----- + +func.func @unsupported_memref(%arg0: memref<32x7xf32, strided<[7, 2]>>, %arg1: memref<32x7xf32, strided<[7, 2]>>, %arg2: memref<32x7xf32, strided<[7, 2]>>) { + // expected-error @below {{attempted to flatten, but failed}} + linalg.map {arith.addf} ins(%arg0, %arg1: memref<32x7xf32, strided<[7, 2]>>, memref<32x7xf32, strided<[7, 2]>>) outs(%arg2: memref<32x7xf32, strided<[7, 2]>>) + return +} + +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) { + %0 = transform.structured.match interface{LinalgOp} in %arg1 : (!transform.any_op) -> !transform.any_op + %flattened = transform.structured.flatten_elementwise %0 + : (!transform.any_op) -> !transform.any_op + transform.yield + } +} -- GitLab From cd8286a667d568c4319b09baa63ba899e3101a19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Danny=20M=C3=B6sch?= Date: Fri, 22 Mar 2024 18:51:44 +0100 Subject: [PATCH 088/306] [GitHub] Allow shortcut for "introductory issue" and request linking to issue in PR (#84635) The answer to many requests in issues to be assigned to users is often "just create a pull request". That's in contradiction to the "introductory issue" instructions posted by the GitHub bot. This change updates the instructions, mentioning the shortcut of "just creating a PR". Moreover, it now explains linking PRs to issues in order to close them automatically upon merge. --- llvm/utils/git/github-automation.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/llvm/utils/git/github-automation.py b/llvm/utils/git/github-automation.py index b21f14eca445..42a658cefac3 100755 --- a/llvm/utils/git/github-automation.py +++ b/llvm/utils/git/github-automation.py @@ -24,12 +24,13 @@ Hi! This issue may be a good introductory issue for people new to working on LLVM. If you would like to work on this issue, your first steps are: -1. In the comments of the issue, request for it to be assigned to you. -2. Fix the issue locally. -3. [Run the test suite](https://llvm.org/docs/TestingGuide.html#unit-and-regression-tests) locally. Remember that the subdirectories under `test/` create fine-grained testing targets, so you can e.g. use `make check-clang-ast` to only run Clang's AST tests. -4. Create a Git commit. -5. Run [`git clang-format HEAD~1`](https://clang.llvm.org/docs/ClangFormat.html#git-integration) to format your changes. -6. Open a [pull request](https://github.com/llvm/llvm-project/pulls) to the [upstream repository](https://github.com/llvm/llvm-project) on GitHub. Detailed instructions can be found [in GitHub's documentation](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request). +1. Check that no other contributor has already been assigned to this issue. If you believe that no one is actually working on it despite an assignment, ping the person. After one week without a response, the assignee may be changed. +1. In the comments of this issue, request for it to be assigned to you, or just create a [pull request](https://github.com/llvm/llvm-project/pulls) after following the steps below. [Mention](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue) this issue in the description of the pull request. +1. Fix the issue locally. +1. [Run the test suite](https://llvm.org/docs/TestingGuide.html#unit-and-regression-tests) locally. Remember that the subdirectories under `test/` create fine-grained testing targets, so you can e.g. use `make check-clang-ast` to only run Clang's AST tests. +1. Create a Git commit. +1. Run [`git clang-format HEAD~1`](https://clang.llvm.org/docs/ClangFormat.html#git-integration) to format your changes. +1. Open a [pull request](https://github.com/llvm/llvm-project/pulls) to the [upstream repository](https://github.com/llvm/llvm-project) on GitHub. Detailed instructions can be found [in GitHub's documentation](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request). [Mention](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue) this issue in the description of the pull request. If you have any further questions about this issue, don't hesitate to ask via a comment in the thread below. """ -- GitLab From fb329f18445cb33d242cc500ca618d03674b22ad Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Fri, 22 Mar 2024 11:15:45 -0700 Subject: [PATCH 089/306] [Target] Move SubRegIdxRanges from MCSubtargetInfo to TargetInfo. (#86245) I'm planning to add HwMode support to SubRegIdxRanges for RISC-V GPR pairs. The MC layer is currently unaware of the HwMode for registers and I'd like to keep it that way. This information is not used by the MC layer so I think it is safe to move it. --- .../include/llvm/CodeGen/TargetRegisterInfo.h | 31 ++++++++++++---- llvm/include/llvm/MC/MCRegisterInfo.h | 21 ----------- llvm/lib/CodeGen/TargetRegisterInfo.cpp | 36 +++++++++++-------- llvm/lib/MC/MCRegisterInfo.cpp | 12 ------- llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp | 4 +-- llvm/unittests/CodeGen/MFCommon.inc | 5 +-- llvm/utils/TableGen/RegisterInfoEmitter.cpp | 29 +++++++-------- 7 files changed, 65 insertions(+), 73 deletions(-) diff --git a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h index 117d3f718297..33c4c745c341 100644 --- a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h +++ b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h @@ -243,9 +243,20 @@ public: unsigned RegSize, SpillSize, SpillAlignment; unsigned VTListOffset; }; + + /// SubRegCoveredBits - Emitted by tablegen: bit range covered by a subreg + /// index, -1 in any being invalid. + struct SubRegCoveredBits { + uint16_t Offset; + uint16_t Size; + }; + private: const TargetRegisterInfoDesc *InfoDesc; // Extra desc array for codegen const char *const *SubRegIndexNames; // Names of subreg indexes. + const SubRegCoveredBits *SubRegIdxRanges; // Pointer to the subreg covered + // bit ranges array. + // Pointer to array of lane masks, one per sub-reg index. const LaneBitmask *SubRegIndexLaneMasks; @@ -256,12 +267,10 @@ private: unsigned HwMode; protected: - TargetRegisterInfo(const TargetRegisterInfoDesc *ID, - regclass_iterator RCB, - regclass_iterator RCE, - const char *const *SRINames, - const LaneBitmask *SRILaneMasks, - LaneBitmask CoveringLanes, + TargetRegisterInfo(const TargetRegisterInfoDesc *ID, regclass_iterator RCB, + regclass_iterator RCE, const char *const *SRINames, + const SubRegCoveredBits *SubIdxRanges, + const LaneBitmask *SRILaneMasks, LaneBitmask CoveringLanes, const RegClassInfo *const RCIs, const MVT::SimpleValueType *const RCVTLists, unsigned Mode = 0); @@ -382,6 +391,16 @@ public: return SubRegIndexNames[SubIdx-1]; } + /// Get the size of the bit range covered by a sub-register index. + /// If the index isn't continuous, return the sum of the sizes of its parts. + /// If the index is used to access subregisters of different sizes, return -1. + unsigned getSubRegIdxSize(unsigned Idx) const; + + /// Get the offset of the bit range covered by a sub-register index. + /// If an Offset doesn't make sense (the index isn't continuous, or is used to + /// access sub-registers at different offsets), return -1. + unsigned getSubRegIdxOffset(unsigned Idx) const; + /// Return a bitmask representing the parts of a register that are covered by /// SubIdx \see LaneBitmask. /// diff --git a/llvm/include/llvm/MC/MCRegisterInfo.h b/llvm/include/llvm/MC/MCRegisterInfo.h index fb4d11ec1d4d..c648ef20fa84 100644 --- a/llvm/include/llvm/MC/MCRegisterInfo.h +++ b/llvm/include/llvm/MC/MCRegisterInfo.h @@ -153,13 +153,6 @@ public: bool operator<(DwarfLLVMRegPair RHS) const { return FromReg < RHS.FromReg; } }; - /// SubRegCoveredBits - Emitted by tablegen: bit range covered by a subreg - /// index, -1 in any being invalid. - struct SubRegCoveredBits { - uint16_t Offset; - uint16_t Size; - }; - private: const MCRegisterDesc *Desc; // Pointer to the descriptor array unsigned NumRegs; // Number of entries in the array @@ -176,8 +169,6 @@ private: const char *RegClassStrings; // Pointer to the class strings. const uint16_t *SubRegIndices; // Pointer to the subreg lookup // array. - const SubRegCoveredBits *SubRegIdxRanges; // Pointer to the subreg covered - // bit ranges array. unsigned NumSubRegIndices; // Number of subreg indices. const uint16_t *RegEncodingTable; // Pointer to array of register // encodings. @@ -278,7 +269,6 @@ public: const int16_t *DL, const LaneBitmask *RUMS, const char *Strings, const char *ClassStrings, const uint16_t *SubIndices, unsigned NumIndices, - const SubRegCoveredBits *SubIdxRanges, const uint16_t *RET) { Desc = D; NumRegs = NR; @@ -294,7 +284,6 @@ public: NumRegUnits = NRU; SubRegIndices = SubIndices; NumSubRegIndices = NumIndices; - SubRegIdxRanges = SubIdxRanges; RegEncodingTable = RET; // Initialize DWARF register mapping variables @@ -387,16 +376,6 @@ public: /// otherwise. unsigned getSubRegIndex(MCRegister RegNo, MCRegister SubRegNo) const; - /// Get the size of the bit range covered by a sub-register index. - /// If the index isn't continuous, return the sum of the sizes of its parts. - /// If the index is used to access subregisters of different sizes, return -1. - unsigned getSubRegIdxSize(unsigned Idx) const; - - /// Get the offset of the bit range covered by a sub-register index. - /// If an Offset doesn't make sense (the index isn't continuous, or is used to - /// access sub-registers at different offsets), return -1. - unsigned getSubRegIdxOffset(unsigned Idx) const; - /// Return the human-readable symbolic target-specific name for the /// specified physical register. const char *getName(MCRegister RegNo) const { diff --git a/llvm/lib/CodeGen/TargetRegisterInfo.cpp b/llvm/lib/CodeGen/TargetRegisterInfo.cpp index c9503fcb77bb..4120c74c23b1 100644 --- a/llvm/lib/CodeGen/TargetRegisterInfo.cpp +++ b/llvm/lib/CodeGen/TargetRegisterInfo.cpp @@ -50,20 +50,16 @@ static cl::opt "high compile time cost in global splitting."), cl::init(5000)); -TargetRegisterInfo::TargetRegisterInfo(const TargetRegisterInfoDesc *ID, - regclass_iterator RCB, regclass_iterator RCE, - const char *const *SRINames, - const LaneBitmask *SRILaneMasks, - LaneBitmask SRICoveringLanes, - const RegClassInfo *const RCIs, - const MVT::SimpleValueType *const RCVTLists, - unsigned Mode) - : InfoDesc(ID), SubRegIndexNames(SRINames), - SubRegIndexLaneMasks(SRILaneMasks), - RegClassBegin(RCB), RegClassEnd(RCE), - CoveringLanes(SRICoveringLanes), - RCInfos(RCIs), RCVTLists(RCVTLists), HwMode(Mode) { -} +TargetRegisterInfo::TargetRegisterInfo( + const TargetRegisterInfoDesc *ID, regclass_iterator RCB, + regclass_iterator RCE, const char *const *SRINames, + const SubRegCoveredBits *SubIdxRanges, const LaneBitmask *SRILaneMasks, + LaneBitmask SRICoveringLanes, const RegClassInfo *const RCIs, + const MVT::SimpleValueType *const RCVTLists, unsigned Mode) + : InfoDesc(ID), SubRegIndexNames(SRINames), SubRegIdxRanges(SubIdxRanges), + SubRegIndexLaneMasks(SRILaneMasks), RegClassBegin(RCB), RegClassEnd(RCE), + CoveringLanes(SRICoveringLanes), RCInfos(RCIs), RCVTLists(RCVTLists), + HwMode(Mode) {} TargetRegisterInfo::~TargetRegisterInfo() = default; @@ -596,6 +592,18 @@ bool TargetRegisterInfo::getCoveringSubRegIndexes( return BestIdx; } +unsigned TargetRegisterInfo::getSubRegIdxSize(unsigned Idx) const { + assert(Idx && Idx < getNumSubRegIndices() && + "This is not a subregister index"); + return SubRegIdxRanges[Idx].Size; +} + +unsigned TargetRegisterInfo::getSubRegIdxOffset(unsigned Idx) const { + assert(Idx && Idx < getNumSubRegIndices() && + "This is not a subregister index"); + return SubRegIdxRanges[Idx].Offset; +} + Register TargetRegisterInfo::lookThruCopyLike(Register SrcReg, const MachineRegisterInfo *MRI) const { diff --git a/llvm/lib/MC/MCRegisterInfo.cpp b/llvm/lib/MC/MCRegisterInfo.cpp index a2c1737e2964..334655616d8d 100644 --- a/llvm/lib/MC/MCRegisterInfo.cpp +++ b/llvm/lib/MC/MCRegisterInfo.cpp @@ -57,18 +57,6 @@ unsigned MCRegisterInfo::getSubRegIndex(MCRegister Reg, return 0; } -unsigned MCRegisterInfo::getSubRegIdxSize(unsigned Idx) const { - assert(Idx && Idx < getNumSubRegIndices() && - "This is not a subregister index"); - return SubRegIdxRanges[Idx].Size; -} - -unsigned MCRegisterInfo::getSubRegIdxOffset(unsigned Idx) const { - assert(Idx && Idx < getNumSubRegIndices() && - "This is not a subregister index"); - return SubRegIdxRanges[Idx].Offset; -} - int MCRegisterInfo::getDwarfRegNum(MCRegister RegNum, bool isEH) const { const DwarfLLVMRegPair *M = isEH ? EHL2DwarfRegs : L2DwarfRegs; unsigned Size = isEH ? EHL2DwarfRegsSize : L2DwarfRegsSize; diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp index 79a7d1cf66c4..245731ad5fc7 100644 --- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp @@ -363,8 +363,8 @@ SIRegisterInfo::SIRegisterInfo(const GCNSubtarget &ST) for (auto &Row : SubRegFromChannelTable) Row.fill(AMDGPU::NoSubRegister); for (unsigned Idx = 1; Idx < getNumSubRegIndices(); ++Idx) { - unsigned Width = AMDGPUSubRegIdxRanges[Idx].Size / 32; - unsigned Offset = AMDGPUSubRegIdxRanges[Idx].Offset / 32; + unsigned Width = getSubRegIdxSize(Idx) / 32; + unsigned Offset = getSubRegIdxOffset(Idx) / 32; assert(Width < SubRegFromChannelTableWidthMap.size()); Width = SubRegFromChannelTableWidthMap[Width]; if (Width == 0) diff --git a/llvm/unittests/CodeGen/MFCommon.inc b/llvm/unittests/CodeGen/MFCommon.inc index 7de7eabdd1f6..1997e8052297 100644 --- a/llvm/unittests/CodeGen/MFCommon.inc +++ b/llvm/unittests/CodeGen/MFCommon.inc @@ -23,9 +23,10 @@ class BogusRegisterInfo : public TargetRegisterInfo { public: BogusRegisterInfo() : TargetRegisterInfo(nullptr, BogusRegisterClasses, BogusRegisterClasses, - nullptr, nullptr, LaneBitmask(~0u), nullptr, nullptr) { + nullptr, nullptr, nullptr, LaneBitmask(~0u), nullptr, + nullptr) { InitMCRegisterInfo(nullptr, 0, 0, 0, nullptr, 0, nullptr, 0, nullptr, - nullptr, nullptr, nullptr, nullptr, 0, nullptr, nullptr); + nullptr, nullptr, nullptr, nullptr, 0, nullptr); } const MCPhysReg * diff --git a/llvm/utils/TableGen/RegisterInfoEmitter.cpp b/llvm/utils/TableGen/RegisterInfoEmitter.cpp index d074e31c6245..c4fc1930488c 100644 --- a/llvm/utils/TableGen/RegisterInfoEmitter.cpp +++ b/llvm/utils/TableGen/RegisterInfoEmitter.cpp @@ -955,16 +955,6 @@ void RegisterInfoEmitter::runMCDesc(raw_ostream &OS, CodeGenTarget &Target, SubRegIdxSeqs.emit(OS, printSubRegIndex); OS << "};\n\n"; - // Emit the table of sub-register index sizes. - OS << "extern const MCRegisterInfo::SubRegCoveredBits " << TargetName - << "SubRegIdxRanges[] = {\n"; - OS << " { " << (uint16_t)-1 << ", " << (uint16_t)-1 << " },\n"; - for (const auto &Idx : SubRegIndices) { - OS << " { " << Idx.Offset << ", " << Idx.Size << " },\t// " - << Idx.getName() << "\n"; - } - OS << "};\n\n"; - // Emit the string table. RegStrings.layout(); RegStrings.emitStringLiteralDef(OS, Twine("extern const char ") + TargetName + @@ -1101,8 +1091,7 @@ void RegisterInfoEmitter::runMCDesc(raw_ostream &OS, CodeGenTarget &Target, << TargetName << "LaneMaskLists, " << TargetName << "RegStrings, " << TargetName << "RegClassStrings, " << TargetName << "SubRegIdxLists, " << (std::distance(SubRegIndices.begin(), SubRegIndices.end()) + 1) << ",\n" - << TargetName << "SubRegIdxRanges, " << TargetName - << "RegEncodingTable);\n\n"; + << TargetName << "RegEncodingTable);\n\n"; EmitRegMapping(OS, Regs, false); @@ -1253,6 +1242,16 @@ void RegisterInfoEmitter::runTargetDesc(raw_ostream &OS, CodeGenTarget &Target, } OS << "\" };\n\n"; + // Emit the table of sub-register index sizes. + OS << "static const TargetRegisterInfo::SubRegCoveredBits " + "SubRegIdxRangeTable[] = {\n"; + OS << " { " << (uint16_t)-1 << ", " << (uint16_t)-1 << " },\n"; + for (const auto &Idx : SubRegIndices) { + OS << " { " << Idx.Offset << ", " << Idx.Size << " },\t// " + << Idx.getName() << "\n"; + } + OS << "};\n\n"; + // Emit SubRegIndex lane masks, including 0. OS << "\nstatic const LaneBitmask SubRegIndexLaneMaskTable[] = {\n " "LaneBitmask::getAll(),\n"; @@ -1634,8 +1633,6 @@ void RegisterInfoEmitter::runTargetDesc(raw_ostream &OS, CodeGenTarget &Target, OS << "extern const char " << TargetName << "RegClassStrings[];\n"; OS << "extern const MCPhysReg " << TargetName << "RegUnitRoots[][2];\n"; OS << "extern const uint16_t " << TargetName << "SubRegIdxLists[];\n"; - OS << "extern const MCRegisterInfo::SubRegCoveredBits " << TargetName - << "SubRegIdxRanges[];\n"; OS << "extern const uint16_t " << TargetName << "RegEncodingTable[];\n"; EmitRegMappingTables(OS, Regs, true); @@ -1646,7 +1643,8 @@ void RegisterInfoEmitter::runTargetDesc(raw_ostream &OS, CodeGenTarget &Target, " unsigned PC, unsigned HwMode)\n" << " : TargetRegisterInfo(&" << TargetName << "RegInfoDesc" << ", RegisterClasses, RegisterClasses+" << RegisterClasses.size() << ",\n" - << " SubRegIndexNameTable, SubRegIndexLaneMaskTable,\n" + << " SubRegIndexNameTable, SubRegIdxRangeTable, " + "SubRegIndexLaneMaskTable,\n" << " "; printMask(OS, RegBank.CoveringLanes); OS << ", RegClassInfos, VTLists, HwMode) {\n" @@ -1661,7 +1659,6 @@ void RegisterInfoEmitter::runTargetDesc(raw_ostream &OS, CodeGenTarget &Target, << " " << TargetName << "RegClassStrings,\n" << " " << TargetName << "SubRegIdxLists,\n" << " " << SubRegIndicesSize + 1 << ",\n" - << " " << TargetName << "SubRegIdxRanges,\n" << " " << TargetName << "RegEncodingTable);\n\n"; EmitRegMapping(OS, Regs, true); -- GitLab From b1575f9082071702bd6aaa2600ce9fe011a091e9 Mon Sep 17 00:00:00 2001 From: Kevin Frei Date: Fri, 22 Mar 2024 11:22:09 -0700 Subject: [PATCH 090/306] Missed a null-ptr check in previous PR for Debuginfod testing (#86292) @GeorgeHuyubo noticed an unchecked shared pointer result in https://github.com/llvm/llvm-project/pull/85693/. This is the fix for that issue. Co-authored-by: Kevin Frei --- lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp b/lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp index 91b8b4a979e0..a9956aa9075f 100644 --- a/lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp +++ b/lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp @@ -53,7 +53,7 @@ static bool IsDwpSymbolFile(const lldb::ModuleSP &module_sp, ObjectFileSP dwp_obj_file = ObjectFile::FindPlugin( module_sp, &file_spec, 0, FileSystem::Instance().GetByteSize(file_spec), dwp_file_data_sp, dwp_file_data_offset); - if (!ObjectFileELF::classof(dwp_obj_file.get())) + if (!dwp_obj_file || !ObjectFileELF::classof(dwp_obj_file.get())) return false; // The presence of a debug_cu_index section is the key identifying feature of // a DWP file. Make sure we don't fill in the section list on dwp_obj_file -- GitLab From 721f149596f27f3d4c5c28ec2a2fac33340fb876 Mon Sep 17 00:00:00 2001 From: Job Henandez Lara Date: Fri, 22 Mar 2024 11:38:51 -0700 Subject: [PATCH 091/306] Fix typo (#86319) Im working on the floating point fmaximum and fminimum functions right now in a different pr and I ran the individual tests by doing ``` ninja libc.test.src.math.smoke._test.__unit__ ``` --- libc/src/math/docs/add_math_function.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libc/src/math/docs/add_math_function.md b/libc/src/math/docs/add_math_function.md index f8bc8a3bdd8b..d1bca222b428 100644 --- a/libc/src/math/docs/add_math_function.md +++ b/libc/src/math/docs/add_math_function.md @@ -177,7 +177,7 @@ implementation (which is very often glibc). - Build and Run a specific unit test: ``` - $ ninja libc.test.src.math._test + $ ninja libc.test.src.math._test.__unit__ $ projects/libc/test/src/math/libc.test.src.math._test ``` -- GitLab From 83e96977cdb6041196366fc01e8abdca52cadb2e Mon Sep 17 00:00:00 2001 From: Vinayak Dev <104419489+vinayakdsci@users.noreply.github.com> Date: Sat, 23 Mar 2024 00:09:53 +0530 Subject: [PATCH 092/306] [libc] Implement strfromd() and strfroml() (#86113) Follow up to #85438. Implements the functions `strfromd()` and `strfroml()` introduced in C23, and unifies the testing framework for `strfrom*()` functions. --- libc/config/linux/x86_64/entrypoints.txt | 2 + libc/spec/stdc.td | 2 + libc/src/stdlib/CMakeLists.txt | 20 ++ libc/src/stdlib/strfromd.cpp | 42 +++ libc/src/stdlib/strfromd.h | 21 ++ libc/src/stdlib/strfromf.h | 2 +- libc/src/stdlib/strfroml.cpp | 47 +++ libc/src/stdlib/strfroml.h | 21 ++ libc/test/src/stdlib/CMakeLists.txt | 31 ++ libc/test/src/stdlib/StrfromTest.h | 435 +++++++++++++++++++++++ libc/test/src/stdlib/strfromd_test.cpp | 13 + libc/test/src/stdlib/strfromf_test.cpp | 98 +---- libc/test/src/stdlib/strfroml_test.cpp | 13 + 13 files changed, 650 insertions(+), 97 deletions(-) create mode 100644 libc/src/stdlib/strfromd.cpp create mode 100644 libc/src/stdlib/strfromd.h create mode 100644 libc/src/stdlib/strfroml.cpp create mode 100644 libc/src/stdlib/strfroml.h create mode 100644 libc/test/src/stdlib/StrfromTest.h create mode 100644 libc/test/src/stdlib/strfromd_test.cpp create mode 100644 libc/test/src/stdlib/strfroml_test.cpp diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt index 8e1ab5cd65f0..8b1cd3fb1052 100644 --- a/libc/config/linux/x86_64/entrypoints.txt +++ b/libc/config/linux/x86_64/entrypoints.txt @@ -180,7 +180,9 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.stdlib.qsort_r libc.src.stdlib.rand libc.src.stdlib.srand + libc.src.stdlib.strfromd libc.src.stdlib.strfromf + libc.src.stdlib.strfroml libc.src.stdlib.strtod libc.src.stdlib.strtof libc.src.stdlib.strtol diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td index 76010a4b4533..3e58e3b88645 100644 --- a/libc/spec/stdc.td +++ b/libc/spec/stdc.td @@ -962,6 +962,8 @@ def StdC : StandardSpec<"stdc"> { FunctionSpec<"srand", RetValSpec, [ArgSpec]>, FunctionSpec<"strfromf", RetValSpec, [ArgSpec, ArgSpec, ArgSpec, ArgSpec]>, + FunctionSpec<"strfromd", RetValSpec, [ArgSpec, ArgSpec, ArgSpec, ArgSpec]>, + FunctionSpec<"strfroml", RetValSpec, [ArgSpec, ArgSpec, ArgSpec, ArgSpec]>, FunctionSpec<"strtof", RetValSpec, [ArgSpec, ArgSpec]>, FunctionSpec<"strtod", RetValSpec, [ArgSpec, ArgSpec]>, diff --git a/libc/src/stdlib/CMakeLists.txt b/libc/src/stdlib/CMakeLists.txt index 22f7f990fb08..a2b16e73f353 100644 --- a/libc/src/stdlib/CMakeLists.txt +++ b/libc/src/stdlib/CMakeLists.txt @@ -62,6 +62,26 @@ add_entrypoint_object( .str_from_util ) +add_entrypoint_object( + strfromd + SRCS + strfromd.cpp + HDRS + strfromd.h + DEPENDS + .str_from_util +) + +add_entrypoint_object( + strfroml + SRCS + strfroml.cpp + HDRS + strfroml.h + DEPENDS + .str_from_util +) + add_header_library( str_from_util HDRS diff --git a/libc/src/stdlib/strfromd.cpp b/libc/src/stdlib/strfromd.cpp new file mode 100644 index 000000000000..1d02a7ad1124 --- /dev/null +++ b/libc/src/stdlib/strfromd.cpp @@ -0,0 +1,42 @@ +//===-- Implementation of strfromd ------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/stdlib/strfromd.h" +#include "src/stdlib/str_from_util.h" + +#include +#include + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(int, strfromd, + (char *__restrict s, size_t n, const char *__restrict format, + double fp)) { + LIBC_ASSERT(s != nullptr); + + printf_core::FormatSection section = + internal::parse_format_string(format, fp); + printf_core::WriteBuffer wb(s, (n > 0 ? n - 1 : 0)); + printf_core::Writer writer(&wb); + + int result = 0; + if (section.has_conv) + result = internal::strfromfloat_convert(&writer, section); + else + result = writer.write(section.raw_string); + + if (result < 0) + return result; + + if (n > 0) + wb.buff[wb.buff_cur] = '\0'; + + return writer.get_chars_written(); +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/stdlib/strfromd.h b/libc/src/stdlib/strfromd.h new file mode 100644 index 000000000000..d2c3fefb6300 --- /dev/null +++ b/libc/src/stdlib/strfromd.h @@ -0,0 +1,21 @@ +//===-- Implementation header for strfromd ------------------------*- C++--===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_STDLIB_STRFROMD_H +#define LLVM_LIBC_SRC_STDLIB_STRFROMD_H + +#include + +namespace LIBC_NAMESPACE { + +int strfromd(char *__restrict s, size_t n, const char *__restrict format, + double fp); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_STDLIB_STRFROMD_H diff --git a/libc/src/stdlib/strfromf.h b/libc/src/stdlib/strfromf.h index b551a58af05a..492c2c33cf08 100644 --- a/libc/src/stdlib/strfromf.h +++ b/libc/src/stdlib/strfromf.h @@ -18,4 +18,4 @@ int strfromf(char *__restrict s, size_t n, const char *__restrict format, } // namespace LIBC_NAMESPACE -#endif // LLVM_LIBC_SRC_STDLIB_STRTOF_H +#endif // LLVM_LIBC_SRC_STDLIB_STRFROMF_H diff --git a/libc/src/stdlib/strfroml.cpp b/libc/src/stdlib/strfroml.cpp new file mode 100644 index 000000000000..6b8781e7c04e --- /dev/null +++ b/libc/src/stdlib/strfroml.cpp @@ -0,0 +1,47 @@ +//===-- Implementation of strfroml ------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/stdlib/strfroml.h" +#include "src/stdlib/str_from_util.h" + +#include +#include + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(int, strfroml, + (char *__restrict s, size_t n, const char *__restrict format, + long double fp)) { + LIBC_ASSERT(s != nullptr); + + printf_core::FormatSection section = + internal::parse_format_string(format, fp); + + // To ensure that the conversion function actually uses long double, + // the length modifier has to be set to LenghtModifier::L + section.length_modifier = printf_core::LengthModifier::L; + + printf_core::WriteBuffer wb(s, (n > 0 ? n - 1 : 0)); + printf_core::Writer writer(&wb); + + int result = 0; + if (section.has_conv) + result = internal::strfromfloat_convert(&writer, section); + else + result = writer.write(section.raw_string); + + if (result < 0) + return result; + + if (n > 0) + wb.buff[wb.buff_cur] = '\0'; + + return writer.get_chars_written(); +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/stdlib/strfroml.h b/libc/src/stdlib/strfroml.h new file mode 100644 index 000000000000..e99d035e4da6 --- /dev/null +++ b/libc/src/stdlib/strfroml.h @@ -0,0 +1,21 @@ +//===-- Implementation header for strfroml ------------------------*- C++--===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_STDLIB_STRFROML_H +#define LLVM_LIBC_SRC_STDLIB_STRFROML_H + +#include + +namespace LIBC_NAMESPACE { + +int strfroml(char *__restrict s, size_t n, const char *__restrict format, + long double fp); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_STDLIB_STRFROML_H diff --git a/libc/test/src/stdlib/CMakeLists.txt b/libc/test/src/stdlib/CMakeLists.txt index cb42bc56f51c..3ccc1cde9193 100644 --- a/libc/test/src/stdlib/CMakeLists.txt +++ b/libc/test/src/stdlib/CMakeLists.txt @@ -168,6 +168,14 @@ add_libc_test( .strtol_test_support ) +add_header_library( + strfrom_test_support + HDRS + StrfromTest.h + DEPENDS + libc.src.__support.CPP.type_traits +) + add_libc_test( strfromf_test SUITE @@ -175,9 +183,32 @@ add_libc_test( SRCS strfromf_test.cpp DEPENDS + .strfrom_test_support libc.src.stdlib.strfromf ) +add_libc_test( + strfromd_test + SUITE + libc-stdlib-tests + SRCS + strfromd_test.cpp + DEPENDS + .strfrom_test_support + libc.src.stdlib.strfromd +) + +add_libc_test( + strfroml_test + SUITE + libc-stdlib-tests + SRCS + strfroml_test.cpp + DEPENDS + .strfrom_test_support + libc.src.stdlib.strfroml +) + add_libc_test( abs_test SUITE diff --git a/libc/test/src/stdlib/StrfromTest.h b/libc/test/src/stdlib/StrfromTest.h new file mode 100644 index 000000000000..f695bbb335bd --- /dev/null +++ b/libc/test/src/stdlib/StrfromTest.h @@ -0,0 +1,435 @@ +//===-- A template class for testing strfrom functions ----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/__support/CPP/type_traits.h" +#include "test/UnitTest/Test.h" + +#define ASSERT_STREQ_LEN(actual_written, actual_str, expected_str) \ + EXPECT_EQ(actual_written, static_cast(sizeof(expected_str) - 1)); \ + EXPECT_STREQ(actual_str, expected_str); + +template +class StrfromTest : public LIBC_NAMESPACE::testing::Test { + + static const bool is_single_prec = + LIBC_NAMESPACE::cpp::is_same::value; + static const bool is_double_prec = + LIBC_NAMESPACE::cpp::is_same::value; + + using FunctionT = int (*)(char *, size_t, const char *, InputT fp); + +public: + void floatDecimalFormat(FunctionT func) { + if (is_single_prec) + floatDecimalSinglePrec(func); + else if (is_double_prec) + floatDecimalDoublePrec(func); + else + floatDecimalLongDoublePrec(func); + } + + void floatHexExpFormat(FunctionT func) { + if (is_single_prec) + floatHexExpSinglePrec(func); + else if (is_double_prec) + floatHexExpDoublePrec(func); + else + floatHexExpLongDoublePrec(func); + } + + void floatDecimalExpFormat(FunctionT func) { + if (is_single_prec) + floatDecimalExpSinglePrec(func); + else if (is_double_prec) + floatDecimalExpDoublePrec(func); + else + floatDecimalExpLongDoublePrec(func); + } + + void floatDecimalAutoFormat(FunctionT func) { + if (is_single_prec) + floatDecimalAutoSinglePrec(func); + else if (is_double_prec) + floatDecimalAutoDoublePrec(func); + else + floatDecimalAutoLongDoublePrec(func); + } + + void improperFormatString(FunctionT func) { + char buff[100]; + int written; + const bool is_long_double = !is_single_prec && !is_double_prec; + + written = func(buff, 37, "A simple string with no conversions.", 1.0); + ASSERT_STREQ_LEN(written, buff, "A simple string with no conversions."); + + written = + func(buff, 37, + "%A simple string with one conversion, should overwrite.", 1.0); + ASSERT_STREQ_LEN(written, buff, is_long_double ? "0X8P-3" : "0X1P+0"); + + written = func(buff, 74, + "A simple string with one conversion in %A " + "between, writes string as it is", + 1.0); + ASSERT_STREQ_LEN(written, buff, + "A simple string with one conversion in %A between, " + "writes string as it is"); + + written = func(buff, 36, "A simple string with one conversion", 1.0); + ASSERT_STREQ_LEN(written, buff, "A simple string with one conversion"); + + written = func(buff, 20, "%1f", 1234567890.0); + ASSERT_STREQ_LEN(written, buff, "%1f"); + } + + void insufficentBufsize(FunctionT func) { + char buff[20]; + int written; + + written = func(buff, 5, "%f", 1234567890.0); + EXPECT_EQ(written, 17); + ASSERT_STREQ(buff, "1234"); + + written = func(buff, 5, "%.5f", 1.05); + EXPECT_EQ(written, 7); + ASSERT_STREQ(buff, "1.05"); + + written = func(buff, 0, "%g", 1.0); + EXPECT_EQ(written, 1); + ASSERT_STREQ(buff, "1.05"); // Make sure that buff has not changed + } + + void floatDecimalSinglePrec(FunctionT func) { + char buff[70]; + int written; + + written = func(buff, 16, "%f", 1.0); + ASSERT_STREQ_LEN(written, buff, "1.000000"); + + written = func(buff, 20, "%f", 1234567890.0); + ASSERT_STREQ_LEN(written, buff, "1234567936.000000"); + + written = func(buff, 67, "%.3f", 1.0); + ASSERT_STREQ_LEN(written, buff, "1.000"); + } + + void floatDecimalDoublePrec(FunctionT func) { + char buff[500]; + int written; + + written = func(buff, 99, "%f", 1.0); + ASSERT_STREQ_LEN(written, buff, "1.000000"); + + written = func(buff, 99, "%F", -1.0); + ASSERT_STREQ_LEN(written, buff, "-1.000000"); + + written = func(buff, 99, "%f", -1.234567); + ASSERT_STREQ_LEN(written, buff, "-1.234567"); + + written = func(buff, 99, "%f", 0.0); + ASSERT_STREQ_LEN(written, buff, "0.000000"); + + written = func(buff, 99, "%f", 1.5); + ASSERT_STREQ_LEN(written, buff, "1.500000"); + + written = func(buff, 499, "%f", 1e300); + ASSERT_STREQ_LEN(written, buff, + "100000000000000005250476025520442024870446858110815915491" + "585411551180245" + "798890819578637137508044786404370444383288387817694252323" + "536043057564479" + "218478670698284838720092657580373783023379478809005936895" + "323497079994508" + "111903896764088007465274278014249457925878882005684283811" + "566947219638686" + "5459400540160.000000"); + + written = func(buff, 99, "%f", 0.1); + ASSERT_STREQ_LEN(written, buff, "0.100000"); + + written = func(buff, 99, "%f", 1234567890123456789.0); + ASSERT_STREQ_LEN(written, buff, "1234567890123456768.000000"); + + written = func(buff, 99, "%f", 9999999999999.99); + ASSERT_STREQ_LEN(written, buff, "9999999999999.990234"); + + written = func(buff, 99, "%f", 0.1); + ASSERT_STREQ_LEN(written, buff, "0.100000"); + + written = func(buff, 99, "%f", 1234567890123456789.0); + ASSERT_STREQ_LEN(written, buff, "1234567890123456768.000000"); + + written = func(buff, 99, "%f", 9999999999999.99); + ASSERT_STREQ_LEN(written, buff, "9999999999999.990234"); + + // Precision Tests + written = func(buff, 100, "%.2f", 9999999999999.99); + ASSERT_STREQ_LEN(written, buff, "9999999999999.99"); + + written = func(buff, 100, "%.1f", 9999999999999.99); + ASSERT_STREQ_LEN(written, buff, "10000000000000.0"); + + written = func(buff, 100, "%.5f", 1.25); + ASSERT_STREQ_LEN(written, buff, "1.25000"); + + written = func(buff, 100, "%.0f", 1.25); + ASSERT_STREQ_LEN(written, buff, "1"); + + written = func(buff, 100, "%.20f", 1.234e-10); + ASSERT_STREQ_LEN(written, buff, "0.00000000012340000000"); + } + + void floatDecimalLongDoublePrec(FunctionT func) { + char buff[45]; + int written; + + written = func(buff, 40, "%f", 1.0L); + ASSERT_STREQ_LEN(written, buff, "1.000000"); + + written = func(buff, 10, "%.f", -2.5L); + ASSERT_STREQ_LEN(written, buff, "-2"); + } + + void floatHexExpSinglePrec(FunctionT func) { + char buff[25]; + int written; + + written = func(buff, 0, "%a", 1234567890.0); + EXPECT_EQ(written, 14); + + written = func(buff, 20, "%a", 1234567890.0); + EXPECT_EQ(written, 14); + ASSERT_STREQ(buff, "0x1.26580cp+30"); + + written = func(buff, 20, "%A", 1234567890.0); + EXPECT_EQ(written, 14); + ASSERT_STREQ(buff, "0X1.26580CP+30"); + } + + void floatHexExpDoublePrec(FunctionT func) { + char buff[60]; + int written; + + written = func(buff, 10, "%a", 1.0); + ASSERT_STREQ_LEN(written, buff, "0x1p+0"); + + written = func(buff, 10, "%A", -1.0); + ASSERT_STREQ_LEN(written, buff, "-0X1P+0"); + + written = func(buff, 30, "%a", -0x1.abcdef12345p0); + ASSERT_STREQ_LEN(written, buff, "-0x1.abcdef12345p+0"); + + written = func(buff, 50, "%A", 0x1.abcdef12345p0); + ASSERT_STREQ_LEN(written, buff, "0X1.ABCDEF12345P+0"); + + written = func(buff, 10, "%a", 0.0); + ASSERT_STREQ_LEN(written, buff, "0x0p+0"); + + written = func(buff, 40, "%a", 1.0e100); + ASSERT_STREQ_LEN(written, buff, "0x1.249ad2594c37dp+332"); + + written = func(buff, 30, "%a", 0.1); + ASSERT_STREQ_LEN(written, buff, "0x1.999999999999ap-4"); + } + + void floatHexExpLongDoublePrec(FunctionT func) { + char buff[55]; + int written; + + written = func(buff, 50, "%a", 0.1L); +#if defined(LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80) + ASSERT_STREQ_LEN(written, buff, "0xc.ccccccccccccccdp-7"); +#elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT64) + ASSERT_STREQ_LEN(written, buff, "0x1.999999999999ap-4"); +#elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT128) + ASSERT_STREQ_LEN(written, buff, "0x1.999999999999999999999999999ap-4"); +#endif + + written = func(buff, 20, "%.1a", 0.1L); +#if defined(LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80) + ASSERT_STREQ_LEN(written, buff, "0xc.dp-7"); +#elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT64) + ASSERT_STREQ_LEN(written, buff, "0x1.ap-4"); +#elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT128) + ASSERT_STREQ_LEN(written, buff, "0x1.ap-4"); +#endif + + written = func(buff, 50, "%a", 1.0e1000L); +#if defined(LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80) + ASSERT_STREQ_LEN(written, buff, "0xf.38db1f9dd3dac05p+3318"); +#elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT64) + ASSERT_STREQ_LEN(written, buff, "inf"); +#elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT128) + ASSERT_STREQ_LEN(written, buff, "0x1.e71b63f3ba7b580af1a52d2a7379p+3321"); +#endif + + written = func(buff, 50, "%a", 1.0e-1000L); +#if defined(LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80) + ASSERT_STREQ_LEN(written, buff, "0x8.68a9188a89e1467p-3325"); +#elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT64) + ASSERT_STREQ_LEN(written, buff, "0x0p+0"); +#elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT128) + ASSERT_STREQ_LEN(written, buff, "0x1.0d152311513c28ce202627c06ec2p-3322"); +#endif + + written = func(buff, 50, "%.1a", 0xf.fffffffffffffffp16380L); +#if defined(LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80) + ASSERT_STREQ_LEN(written, buff, "0x1.0p+16384"); +#elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT64) + ASSERT_STREQ_LEN(written, buff, "inf"); +#elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT128) + ASSERT_STREQ_LEN(written, buff, "0x2.0p+16383"); +#endif + } + + void floatDecimalExpSinglePrec(FunctionT func) { + char buff[25]; + int written; + + written = func(buff, 20, "%.9e", 1234567890.0); + ASSERT_STREQ_LEN(written, buff, "1.234567936e+09"); + + written = func(buff, 20, "%.9E", 1234567890.0); + ASSERT_STREQ_LEN(written, buff, "1.234567936E+09"); + } + + void floatDecimalExpDoublePrec(FunctionT func) { + char buff[101]; + int written; + + written = func(buff, 100, "%e", 1.0); + ASSERT_STREQ_LEN(written, buff, "1.000000e+00"); + + written = func(buff, 100, "%E", -1.0); + ASSERT_STREQ_LEN(written, buff, "-1.000000E+00"); + + written = func(buff, 100, "%e", -1.234567); + ASSERT_STREQ_LEN(written, buff, "-1.234567e+00"); + + written = func(buff, 100, "%e", 0.0); + ASSERT_STREQ_LEN(written, buff, "0.000000e+00"); + + written = func(buff, 100, "%e", 1.5); + ASSERT_STREQ_LEN(written, buff, "1.500000e+00"); + + written = func(buff, 100, "%e", 1e300); + ASSERT_STREQ_LEN(written, buff, "1.000000e+300"); + + written = func(buff, 100, "%e", 1234567890123456789.0); + ASSERT_STREQ_LEN(written, buff, "1.234568e+18"); + + // Precision Tests + written = func(buff, 100, "%.1e", 1.0); + ASSERT_STREQ_LEN(written, buff, "1.0e+00"); + + written = func(buff, 100, "%.1e", 1.99); + ASSERT_STREQ_LEN(written, buff, "2.0e+00"); + + written = func(buff, 100, "%.1e", 9.99); + ASSERT_STREQ_LEN(written, buff, "1.0e+01"); + } + + void floatDecimalExpLongDoublePrec(FunctionT func) { + char buff[100]; + int written; + +#if defined(LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80) + written = func(buff, 90, "%.9e", 1000000000500000000.1L); + ASSERT_STREQ_LEN(written, buff, "1.000000001e+18"); + + written = func(buff, 90, "%.9e", 1000000000500000000.0L); + ASSERT_STREQ_LEN(written, buff, "1.000000000e+18"); + + written = func(buff, 90, "%e", 0xf.fffffffffffffffp+16380L); + ASSERT_STREQ_LEN(written, buff, "1.189731e+4932"); +#endif // LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80 + } + + void floatDecimalAutoSinglePrec(FunctionT func) { + char buff[25]; + int written; + + written = func(buff, 20, "%.9g", 1234567890.0); + ASSERT_STREQ_LEN(written, buff, "1.23456794e+09"); + + written = func(buff, 20, "%.9G", 1234567890.0); + ASSERT_STREQ_LEN(written, buff, "1.23456794E+09"); + } + + void floatDecimalAutoDoublePrec(FunctionT func) { + char buff[120]; + int written; + + written = func(buff, 100, "%g", 1234567890123456789.0); + ASSERT_STREQ_LEN(written, buff, "1.23457e+18"); + + written = func(buff, 100, "%g", 9999990000000.00); + ASSERT_STREQ_LEN(written, buff, "9.99999e+12"); + + written = func(buff, 100, "%g", 9999999000000.00); + ASSERT_STREQ_LEN(written, buff, "1e+13"); + + written = func(buff, 100, "%g", 0xa.aaaaaaaaaaaaaabp-7); + ASSERT_STREQ_LEN(written, buff, "0.0833333"); + + written = func(buff, 100, "%g", 0.00001); + ASSERT_STREQ_LEN(written, buff, "1e-05"); + + // Precision Tests + written = func(buff, 100, "%.0g", 0.0); + ASSERT_STREQ_LEN(written, buff, "0"); + + written = func(buff, 100, "%.2g", 0.1); + ASSERT_STREQ_LEN(written, buff, "0.1"); + + written = func(buff, 100, "%.2g", 1.09); + ASSERT_STREQ_LEN(written, buff, "1.1"); + + written = func(buff, 100, "%.15g", 22.25); + ASSERT_STREQ_LEN(written, buff, "22.25"); + + written = func(buff, 100, "%.20g", 1.234e-10); + ASSERT_STREQ_LEN(written, buff, "1.2340000000000000814e-10"); + } + + void floatDecimalAutoLongDoublePrec(FunctionT func) { + char buff[100]; + int written; + +#if defined(LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80) + written = func(buff, 99, "%g", 0xf.fffffffffffffffp+16380L); + ASSERT_STREQ_LEN(written, buff, "1.18973e+4932"); + + written = func(buff, 99, "%g", 0xa.aaaaaaaaaaaaaabp-7L); + ASSERT_STREQ_LEN(written, buff, "0.0833333"); + + written = func(buff, 99, "%g", 9.99999999999e-100L); + ASSERT_STREQ_LEN(written, buff, "1e-99"); +#endif // LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80 + } +}; + +#define STRFROM_TEST(InputType, name, func) \ + using LlvmLibc##name##Test = StrfromTest; \ + TEST_F(LlvmLibc##name##Test, FloatDecimalFormat) { \ + floatDecimalFormat(func); \ + } \ + TEST_F(LlvmLibc##name##Test, FloatHexExpFormat) { floatHexExpFormat(func); } \ + TEST_F(LlvmLibc##name##Test, FloatDecimalAutoFormat) { \ + floatDecimalAutoFormat(func); \ + } \ + TEST_F(LlvmLibc##name##Test, FloatDecimalExpFormat) { \ + floatDecimalExpFormat(func); \ + } \ + TEST_F(LlvmLibc##name##Test, ImproperFormatString) { \ + improperFormatString(func); \ + } \ + TEST_F(LlvmLibc##name##Test, InsufficientBufferSize) { \ + insufficentBufsize(func); \ + } diff --git a/libc/test/src/stdlib/strfromd_test.cpp b/libc/test/src/stdlib/strfromd_test.cpp new file mode 100644 index 000000000000..55724d7e902b --- /dev/null +++ b/libc/test/src/stdlib/strfromd_test.cpp @@ -0,0 +1,13 @@ +//===-- Unittests for strfromd --------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "StrfromTest.h" +#include "src/stdlib/strfromd.h" +#include "test/UnitTest/Test.h" + +STRFROM_TEST(double, Strfromd, LIBC_NAMESPACE::strfromd) diff --git a/libc/test/src/stdlib/strfromf_test.cpp b/libc/test/src/stdlib/strfromf_test.cpp index c5489f5f3af2..8b987fd434ac 100644 --- a/libc/test/src/stdlib/strfromf_test.cpp +++ b/libc/test/src/stdlib/strfromf_test.cpp @@ -6,102 +6,8 @@ // //===----------------------------------------------------------------------===// +#include "StrfromTest.h" #include "src/stdlib/strfromf.h" #include "test/UnitTest/Test.h" -TEST(LlvmLibcStrfromfTest, DecimalFloatFormat) { - char buff[100]; - int written; - - written = LIBC_NAMESPACE::strfromf(buff, 16, "%f", 1.0); - EXPECT_EQ(written, 8); - ASSERT_STREQ(buff, "1.000000"); - - written = LIBC_NAMESPACE::strfromf(buff, 20, "%f", 1234567890.0); - EXPECT_EQ(written, 17); - ASSERT_STREQ(buff, "1234567936.000000"); - - written = LIBC_NAMESPACE::strfromf(buff, 5, "%f", 1234567890.0); - EXPECT_EQ(written, 17); - ASSERT_STREQ(buff, "1234"); - - written = LIBC_NAMESPACE::strfromf(buff, 67, "%.3f", 1.0); - EXPECT_EQ(written, 5); - ASSERT_STREQ(buff, "1.000"); - - written = LIBC_NAMESPACE::strfromf(buff, 20, "%1f", 1234567890.0); - EXPECT_EQ(written, 3); - ASSERT_STREQ(buff, "%1f"); -} - -TEST(LlvmLibcStrfromfTest, HexExpFloatFormat) { - char buff[100]; - int written; - - written = LIBC_NAMESPACE::strfromf(buff, 0, "%a", 1234567890.0); - EXPECT_EQ(written, 14); - - written = LIBC_NAMESPACE::strfromf(buff, 20, "%a", 1234567890.0); - EXPECT_EQ(written, 14); - ASSERT_STREQ(buff, "0x1.26580cp+30"); - - written = LIBC_NAMESPACE::strfromf(buff, 20, "%A", 1234567890.0); - EXPECT_EQ(written, 14); - ASSERT_STREQ(buff, "0X1.26580CP+30"); -} - -TEST(LlvmLibcStrfromfTest, DecimalExpFloatFormat) { - char buff[100]; - int written; - written = LIBC_NAMESPACE::strfromf(buff, 20, "%.9e", 1234567890.0); - EXPECT_EQ(written, 15); - ASSERT_STREQ(buff, "1.234567936e+09"); - - written = LIBC_NAMESPACE::strfromf(buff, 20, "%.9E", 1234567890.0); - EXPECT_EQ(written, 15); - ASSERT_STREQ(buff, "1.234567936E+09"); -} - -TEST(LlvmLibcStrfromfTest, AutoDecimalFloatFormat) { - char buff[100]; - int written; - - written = LIBC_NAMESPACE::strfromf(buff, 20, "%.9g", 1234567890.0); - EXPECT_EQ(written, 14); - ASSERT_STREQ(buff, "1.23456794e+09"); - - written = LIBC_NAMESPACE::strfromf(buff, 20, "%.9G", 1234567890.0); - EXPECT_EQ(written, 14); - ASSERT_STREQ(buff, "1.23456794E+09"); - - written = LIBC_NAMESPACE::strfromf(buff, 0, "%G", 1.0); - EXPECT_EQ(written, 1); -} - -TEST(LlvmLibcStrfromfTest, ImproperFormatString) { - - char buff[100]; - int retval; - retval = LIBC_NAMESPACE::strfromf( - buff, 37, "A simple string with no conversions.", 1.0); - EXPECT_EQ(retval, 36); - ASSERT_STREQ(buff, "A simple string with no conversions."); - - retval = LIBC_NAMESPACE::strfromf( - buff, 37, "%A simple string with one conversion, should overwrite.", 1.0); - EXPECT_EQ(retval, 6); - ASSERT_STREQ(buff, "0X1P+0"); - - retval = LIBC_NAMESPACE::strfromf(buff, 74, - "A simple string with one conversion in %A " - "between, writes string as it is", - 1.0); - EXPECT_EQ(retval, 73); - ASSERT_STREQ(buff, "A simple string with one conversion in %A between, " - "writes string as it is"); - - retval = LIBC_NAMESPACE::strfromf(buff, 36, - "A simple string with one conversion", 1.0); - EXPECT_EQ(retval, 35); - ASSERT_STREQ(buff, "A simple string with one conversion"); -} +STRFROM_TEST(float, StrFromf, LIBC_NAMESPACE::strfromf) diff --git a/libc/test/src/stdlib/strfroml_test.cpp b/libc/test/src/stdlib/strfroml_test.cpp new file mode 100644 index 000000000000..cf472a39a5bf --- /dev/null +++ b/libc/test/src/stdlib/strfroml_test.cpp @@ -0,0 +1,13 @@ +//===-- Unittests for strfroml --------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "StrfromTest.h" +#include "src/stdlib/strfroml.h" +#include "test/UnitTest/Test.h" + +STRFROM_TEST(long double, Strfroml, LIBC_NAMESPACE::strfroml) -- GitLab From e64e15ee597370a9731fcba0b2b8a514f26125e7 Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Fri, 22 Mar 2024 11:39:42 -0700 Subject: [PATCH 093/306] [RISCV] Move the RISCVSchedule.td include after RISCVRegisterInfo.td. NFC Registers shouldn't depend on the scheduler, but a scheduler predicate could depend on a register. This would make it possible to move VLDSX0Pred out of the SiFive7 scheduler model to RISCVSchedule.td if another model needed it. --- llvm/lib/Target/RISCV/RISCV.td | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/Target/RISCV/RISCV.td b/llvm/lib/Target/RISCV/RISCV.td index 22736edc5f07..9fb84efd5b6f 100644 --- a/llvm/lib/Target/RISCV/RISCV.td +++ b/llvm/lib/Target/RISCV/RISCV.td @@ -24,8 +24,8 @@ include "RISCVSystemOperands.td" // Registers, calling conventions, instruction descriptions. //===----------------------------------------------------------------------===// -include "RISCVSchedule.td" include "RISCVRegisterInfo.td" +include "RISCVSchedule.td" include "RISCVCallingConv.td" include "RISCVInstrInfo.td" include "GISel/RISCVRegisterBanks.td" -- GitLab From 2120f574103c487787390263b3692c4b167f6bdf Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Fri, 22 Mar 2024 11:45:51 -0700 Subject: [PATCH 094/306] Reapply [workflows] Split pr-code-format into two parts to make it more secure (#78215) (#80495) Actions triggered by pull_request_target events have access to all repository secrets, so it is unsafe to use them when executing untrusted code. The pr-code-format workflow does not execute any untrusted code, but it passes untrused input into clang-format. An attacker could use this to exploit a flaw in clang-format and potentially gain access to the repository secrets. By splitting the workflow, we can use the pull_request target which is more secure and isolate the issue write permissions in a separate job. The pull_request target also makes it easier to test changes to the code-format-helepr.py script, because the version of the script from the pull request will be used rather than the version of the script from main. Fixes #77142 --- .github/workflows/issue-write.yml | 128 +++++++++++++++++++++++++++ .github/workflows/pr-code-format.yml | 20 +++-- llvm/utils/git/code-format-helper.py | 26 ++++++ 3 files changed, 167 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/issue-write.yml diff --git a/.github/workflows/issue-write.yml b/.github/workflows/issue-write.yml new file mode 100644 index 000000000000..02a5f7c213e8 --- /dev/null +++ b/.github/workflows/issue-write.yml @@ -0,0 +1,128 @@ +name: Comment on an issue + +on: + workflow_run: + workflows: ["Check code formatting"] + types: + - completed + +permissions: + contents: read + +jobs: + pr-comment: + runs-on: ubuntu-latest + permissions: + pull-requests: write + if: > + github.event.workflow_run.event == 'pull_request' + steps: + - name: 'Download artifact' + uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1 + with: + github-token: ${{ secrets.ISSUE_WRITE_DOWNLOAD_ARTIFACT }} + run-id: ${{ github.event.workflow_run.id }} + name: workflow-args + + - name: 'Comment on PR' + uses: actions/github-script@v3 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + var fs = require('fs'); + const comments = JSON.parse(fs.readFileSync('./comments')); + if (!comments) { + return; + } + + let runInfo = await github.actions.getWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.payload.workflow_run.id + }); + + console.log(runInfo); + + + // Query to find the number of the pull request that triggered this job. + // The associated pull requests are based off of the branch name, so if + // you create a pull request for a branch, close it, and then create + // another pull request with the same branch, then this query will return + // two associated pull requests. This is why we have to fetch all the + // associated pull requests and then iterate through them to find the + // one that is open. + const gql_query = ` + query($repo_owner : String!, $repo_name : String!, $branch: String!) { + repository(owner: $repo_owner, name: $repo_name) { + ref (qualifiedName: $branch) { + associatedPullRequests(first: 100) { + nodes { + baseRepository { + owner { + login + } + } + number + state + } + } + } + } + } + ` + const gql_variables = { + repo_owner: runInfo.data.head_repository.owner.login, + repo_name: runInfo.data.head_repository.name, + branch: runInfo.data.head_branch + } + const gql_result = await github.graphql(gql_query, gql_variables); + console.log(gql_result); + console.log(gql_result.repository.ref.associatedPullRequests.nodes); + + var pr_number = 0; + gql_result.repository.ref.associatedPullRequests.nodes.forEach((pr) => { + if (pr.baseRepository.owner.login = context.repo.owner && pr.state == 'OPEN') { + pr_number = pr.number; + } + }); + if (pr_number == 0) { + console.log("Error retrieving pull request number"); + return; + } + + await comments.forEach(function (comment) { + if (comment.id) { + // Security check: Ensure that this comment was created by + // the github-actions bot, so a malicious input won't overwrite + // a user's comment. + github.issues.getComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: comment.id + }).then((old_comment) => { + console.log(old_comment); + if (old_comment.data.user.login != "github-actions[bot]") { + console.log("Invalid comment id: " + comment.id); + return; + } + github.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pr_number, + comment_id: comment.id, + body: comment.body + }); + }); + } else { + github.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pr_number, + body: comment.body + }); + } + }); + + - name: Dump comments file + if: always() + run: cat comments diff --git a/.github/workflows/pr-code-format.yml b/.github/workflows/pr-code-format.yml index 1d1fa2483b65..2ed9b05cac12 100644 --- a/.github/workflows/pr-code-format.yml +++ b/.github/workflows/pr-code-format.yml @@ -1,12 +1,9 @@ name: "Check code formatting" on: - pull_request_target: + pull_request: branches: - main -permissions: - pull-requests: write - jobs: code_formatter: runs-on: ubuntu-latest @@ -31,12 +28,13 @@ jobs: separator: "," skip_initial_fetch: true - # We need to make sure that we aren't executing/using any code from the - # PR for security reasons as we're using pull_request_target. Checkout - # the target branch with the necessary files. + # We need to pull the script from the main branch, so that we ensure + # we get the latest version of this script. - name: Fetch code formatting utils uses: actions/checkout@v4 with: + reository: ${{ github.repository }} + ref: ${{ github.base_ref }} sparse-checkout: | llvm/utils/git/requirements_formatting.txt llvm/utils/git/code-format-helper.py @@ -77,8 +75,16 @@ jobs: # the merge base. run: | python ./code-format-tools/llvm/utils/git/code-format-helper.py \ + --write-comment-to-file \ --token ${{ secrets.GITHUB_TOKEN }} \ --issue-number $GITHUB_PR_NUMBER \ --start-rev $(git merge-base $START_REV $END_REV) \ --end-rev $END_REV \ --changed-files "$CHANGED_FILES" + + - uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0 + if: always() + with: + name: workflow-args + path: | + comments diff --git a/llvm/utils/git/code-format-helper.py b/llvm/utils/git/code-format-helper.py index 1113bf02570b..af1bb3b5aec5 100755 --- a/llvm/utils/git/code-format-helper.py +++ b/llvm/utils/git/code-format-helper.py @@ -44,6 +44,7 @@ class FormatArgs: token: str = None verbose: bool = True issue_number: int = 0 + write_comment_to_file: bool = False def __init__(self, args: argparse.Namespace = None) -> None: if not args is None: @@ -53,12 +54,14 @@ class FormatArgs: self.token = args.token self.changed_files = args.changed_files self.issue_number = args.issue_number + self.write_comment_to_file = args.write_comment_to_file class FormatHelper: COMMENT_TAG = "" name: str friendly_name: str + comment: dict = None @property def comment_tag(self) -> str: @@ -119,6 +122,13 @@ View the diff from {self.name} here. comment_text = self.comment_tag + "\n\n" + comment_text existing_comment = self.find_comment(pr) + + if args.write_comment_to_file: + self.comment = {"body": comment_text} + if existing_comment: + self.comment["id"] = existing_comment.id + return + if existing_comment: existing_comment.edit(comment_text) elif create_new: @@ -310,6 +320,8 @@ def hook_main(): if fmt.has_tool(): if not fmt.run(args.changed_files, args): failed_fmts.append(fmt.name) + if fmt.comment: + comments.append(fmt.comment) else: print(f"Couldn't find {fmt.name}, can't check " + fmt.friendly_name.lower()) @@ -350,6 +362,11 @@ if __name__ == "__main__": type=str, help="Comma separated list of files that has been changed", ) + parser.add_argument( + "--write-comment-to-file", + action="store_true", + help="Don't post comments on the PR, instead write the comments and metadata a file called 'comment'", + ) args = FormatArgs(parser.parse_args()) @@ -358,9 +375,18 @@ if __name__ == "__main__": changed_files = args.changed_files.split(",") failed_formatters = [] + comments = [] for fmt in ALL_FORMATTERS: if not fmt.run(changed_files, args): failed_formatters.append(fmt.name) + if fmt.comment: + comments.append(fmt.comment) + + if len(comments): + with open("comments", "w") as f: + import json + + json.dump(comments, f) if len(failed_formatters) > 0: print(f"error: some formatters failed: {' '.join(failed_formatters)}") -- GitLab From 80fc61270d6cae8d1bfbd9211727fe1d22fc0cd5 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Fri, 22 Mar 2024 11:46:06 -0700 Subject: [PATCH 095/306] Revert "[bazel] Update to 7.x (#86297)" (#86325) Reverting for https://github.com/llvm/llvm-project/pull/86297#issuecomment-2015660662 This reverts commit ab8ace3bfd5165a8532f710f9c2d8dd40c3fac39. --- utils/bazel/.bazelrc | 5 ----- utils/bazel/.bazelversion | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/utils/bazel/.bazelrc b/utils/bazel/.bazelrc index e8d055ec2322..1d7cf4a4df1b 100644 --- a/utils/bazel/.bazelrc +++ b/utils/bazel/.bazelrc @@ -6,11 +6,6 @@ # Common flags that apply to all configurations. # Use sparingly for things common to all compilers and platforms. ############################################################################### - -# Flip off to disable MODULE.bazel until we're ready. -# https://github.com/llvm/llvm-project/issues/55924 -common --enable_bzlmod=false - # Prevent invalid caching if input files are modified during a build. build --experimental_guard_against_concurrent_changes diff --git a/utils/bazel/.bazelversion b/utils/bazel/.bazelversion index 21c8c7b46b89..5e3254243a3b 100644 --- a/utils/bazel/.bazelversion +++ b/utils/bazel/.bazelversion @@ -1 +1 @@ -7.1.1 +6.1.2 -- GitLab From 50e6218132d6d9cfc2645fa19cf72d0437a62d95 Mon Sep 17 00:00:00 2001 From: Aiden Grossman Date: Fri, 22 Mar 2024 11:42:15 -0700 Subject: [PATCH 096/306] Reland "[llvm-exegesis] Add thread IDs to subprocess memory names (#84451)" This reverts commit 1fe9c417a0bf143f9bb9f9e1fbf7b20f44196883. This relands commit 6bbe8a296ee91754d423c59c35727eaa624f7140. This was causing build failures on one of the ARMv8 builders. Still not completely sure why, but relanding it to see if the failure pops up again. If it does, the plan is to fix forward by disabling tests on ARM temporarily as llvm-exegesis does not currently use SubprocessMemory on ARM. --- .../llvm-exegesis/lib/BenchmarkRunner.cpp | 9 +++--- .../llvm-exegesis/lib/SubprocessMemory.cpp | 30 +++++++++++++------ .../llvm-exegesis/lib/SubprocessMemory.h | 5 +++- .../X86/SubprocessMemoryTest.cpp | 5 +++- 4 files changed, 34 insertions(+), 15 deletions(-) diff --git a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp index 6d5f0286fb9c..498308e2edbe 100644 --- a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp +++ b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp @@ -405,6 +405,7 @@ private: if (AddMemDefError) return AddMemDefError; + long ParentTID = SubprocessMemory::getCurrentTID(); pid_t ParentOrChildPID = fork(); if (ParentOrChildPID == -1) { @@ -418,7 +419,7 @@ private: // Unregister handlers, signal handling is now handled through ptrace in // the host process. sys::unregisterHandlers(); - runChildSubprocess(PipeFiles[0], Key); + runChildSubprocess(PipeFiles[0], Key, ParentTID); // The child process terminates in the above function, so we should never // get to this point. llvm_unreachable("Child process didn't exit when expected."); @@ -439,8 +440,8 @@ private: setrlimit(RLIMIT_CORE, &rlim); } - [[noreturn]] void runChildSubprocess(int Pipe, - const BenchmarkKey &Key) const { + [[noreturn]] void runChildSubprocess(int Pipe, const BenchmarkKey &Key, + long ParentTID) const { // Disable core dumps in the child process as otherwise everytime we // encounter an execution failure like a segmentation fault, we will create // a core dump. We report the information directly rather than require the @@ -497,7 +498,7 @@ private: Expected AuxMemFDOrError = SubprocessMemory::setupAuxiliaryMemoryInSubprocess( - Key.MemoryValues, ParentPID, CounterFileDescriptor); + Key.MemoryValues, ParentPID, ParentTID, CounterFileDescriptor); if (!AuxMemFDOrError) exit(ChildProcessExitCodeE::AuxiliaryMemorySetupFailed); diff --git a/llvm/tools/llvm-exegesis/lib/SubprocessMemory.cpp b/llvm/tools/llvm-exegesis/lib/SubprocessMemory.cpp index a49fa077257d..1fd81bd407be 100644 --- a/llvm/tools/llvm-exegesis/lib/SubprocessMemory.cpp +++ b/llvm/tools/llvm-exegesis/lib/SubprocessMemory.cpp @@ -9,11 +9,13 @@ #include "SubprocessMemory.h" #include "Error.h" #include "llvm/Support/Error.h" +#include "llvm/Support/FormatVariadic.h" #include #ifdef __linux__ #include #include +#include #include #endif @@ -22,12 +24,21 @@ namespace exegesis { #if defined(__linux__) && !defined(__ANDROID__) +long SubprocessMemory::getCurrentTID() { + // We're using the raw syscall here rather than the gettid() function provided + // by most libcs for compatibility as gettid() was only added to glibc in + // version 2.30. + return syscall(SYS_gettid); +} + Error SubprocessMemory::initializeSubprocessMemory(pid_t ProcessID) { // Add the PID to the shared memory name so that if we're running multiple // processes at the same time, they won't interfere with each other. // This comes up particularly often when running the exegesis tests with - // llvm-lit - std::string AuxiliaryMemoryName = "/auxmem" + std::to_string(ProcessID); + // llvm-lit. Additionally add the TID so that downstream consumers + // using multiple threads don't run into conflicts. + std::string AuxiliaryMemoryName = + formatv("/{0}auxmem{1}", getCurrentTID(), ProcessID); int AuxiliaryMemoryFD = shm_open(AuxiliaryMemoryName.c_str(), O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); if (AuxiliaryMemoryFD == -1) @@ -47,8 +58,8 @@ Error SubprocessMemory::addMemoryDefinition( pid_t ProcessPID) { SharedMemoryNames.reserve(MemoryDefinitions.size()); for (auto &[Name, MemVal] : MemoryDefinitions) { - std::string SharedMemoryName = "/" + std::to_string(ProcessPID) + "memdef" + - std::to_string(MemVal.Index); + std::string SharedMemoryName = + formatv("/{0}t{1}memdef{2}", ProcessPID, getCurrentTID(), MemVal.Index); SharedMemoryNames.push_back(SharedMemoryName); int SharedMemoryFD = shm_open(SharedMemoryName.c_str(), O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); @@ -82,8 +93,9 @@ Error SubprocessMemory::addMemoryDefinition( Expected SubprocessMemory::setupAuxiliaryMemoryInSubprocess( std::unordered_map MemoryDefinitions, - pid_t ParentPID, int CounterFileDescriptor) { - std::string AuxiliaryMemoryName = "/auxmem" + std::to_string(ParentPID); + pid_t ParentPID, long ParentTID, int CounterFileDescriptor) { + std::string AuxiliaryMemoryName = + formatv("/{0}auxmem{1}", ParentTID, ParentPID); int AuxiliaryMemoryFileDescriptor = shm_open(AuxiliaryMemoryName.c_str(), O_RDWR, S_IRUSR | S_IWUSR); if (AuxiliaryMemoryFileDescriptor == -1) @@ -97,8 +109,8 @@ Expected SubprocessMemory::setupAuxiliaryMemoryInSubprocess( return make_error("Mapping auxiliary memory failed"); AuxiliaryMemoryMapping[0] = CounterFileDescriptor; for (auto &[Name, MemVal] : MemoryDefinitions) { - std::string MemoryValueName = "/" + std::to_string(ParentPID) + "memdef" + - std::to_string(MemVal.Index); + std::string MemoryValueName = + formatv("/{0}t{1}memdef{2}", ParentPID, ParentTID, MemVal.Index); AuxiliaryMemoryMapping[AuxiliaryMemoryOffset + MemVal.Index] = shm_open(MemoryValueName.c_str(), O_RDWR, S_IRUSR | S_IWUSR); if (AuxiliaryMemoryMapping[AuxiliaryMemoryOffset + MemVal.Index] == -1) @@ -133,7 +145,7 @@ Error SubprocessMemory::addMemoryDefinition( Expected SubprocessMemory::setupAuxiliaryMemoryInSubprocess( std::unordered_map MemoryDefinitions, - pid_t ParentPID, int CounterFileDescriptor) { + pid_t ParentPID, long ParentTID, int CounterFileDescriptor) { return make_error( "setupAuxiliaryMemoryInSubprocess is only supported on Linux"); } diff --git a/llvm/tools/llvm-exegesis/lib/SubprocessMemory.h b/llvm/tools/llvm-exegesis/lib/SubprocessMemory.h index e20b50cdc811..572d1085d9cf 100644 --- a/llvm/tools/llvm-exegesis/lib/SubprocessMemory.h +++ b/llvm/tools/llvm-exegesis/lib/SubprocessMemory.h @@ -35,6 +35,9 @@ public: static constexpr const size_t AuxiliaryMemoryOffset = 1; static constexpr const size_t AuxiliaryMemorySize = 4096; + // Gets the thread ID for the calling thread. + static long getCurrentTID(); + Error initializeSubprocessMemory(pid_t ProcessID); // The following function sets up memory definitions. It creates shared @@ -54,7 +57,7 @@ public: // section. static Expected setupAuxiliaryMemoryInSubprocess( std::unordered_map MemoryDefinitions, - pid_t ParentPID, int CounterFileDescriptor); + pid_t ParentPID, long ParentTID, int CounterFileDescriptor); ~SubprocessMemory(); diff --git a/llvm/unittests/tools/llvm-exegesis/X86/SubprocessMemoryTest.cpp b/llvm/unittests/tools/llvm-exegesis/X86/SubprocessMemoryTest.cpp index c07ec188a602..7c23e7b7e9c5 100644 --- a/llvm/unittests/tools/llvm-exegesis/X86/SubprocessMemoryTest.cpp +++ b/llvm/unittests/tools/llvm-exegesis/X86/SubprocessMemoryTest.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #endif // __linux__ @@ -49,7 +50,9 @@ protected: std::string getSharedMemoryName(const unsigned TestNumber, const unsigned DefinitionNumber) { - return "/" + std::to_string(getSharedMemoryNumber(TestNumber)) + "memdef" + + long CurrentTID = syscall(SYS_gettid); + return "/" + std::to_string(getSharedMemoryNumber(TestNumber)) + "t" + + std::to_string(CurrentTID) + "memdef" + std::to_string(DefinitionNumber); } -- GitLab From 4b907414d2c0445ac1ae11811aaa4aaa2e060cf9 Mon Sep 17 00:00:00 2001 From: Ulrich Weigand Date: Fri, 22 Mar 2024 19:59:22 +0100 Subject: [PATCH 097/306] [SystemZ] Add support for llvm.readcyclecounter The llvm.readcyclecounter intrinsic can be implemented via the STORE CLOCK FAST (STCKF) instruction. --- .../Target/SystemZ/SystemZISelLowering.cpp | 38 +++++++++++++++++++ llvm/lib/Target/SystemZ/SystemZISelLowering.h | 4 ++ llvm/lib/Target/SystemZ/SystemZInstrSystem.td | 2 +- llvm/lib/Target/SystemZ/SystemZOperators.td | 4 ++ llvm/test/CodeGen/SystemZ/readcyclecounter.ll | 27 +++++++++++++ 5 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 llvm/test/CodeGen/SystemZ/readcyclecounter.ll diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp index bc60d14ee700..da4bcd7f0c66 100644 --- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp +++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp @@ -372,6 +372,9 @@ SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &TM, // Handle prefetches with PFD or PFDRL. setOperationAction(ISD::PREFETCH, MVT::Other, Custom); + // Handle readcyclecounter with STCKF. + setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Custom); + for (MVT VT : MVT::fixedlen_vector_valuetypes()) { // Assume by default that all vector operations need to be expanded. for (unsigned Opcode = 0; Opcode < ISD::BUILTIN_OP_END; ++Opcode) @@ -6077,6 +6080,27 @@ SDValue SystemZTargetLowering::lowerIS_FPCLASS(SDValue Op, return getCCResult(DAG, Intr); } +SDValue SystemZTargetLowering::lowerREADCYCLECOUNTER(SDValue Op, + SelectionDAG &DAG) const { + SDLoc DL(Op); + SDValue Chain = Op.getOperand(0); + + // STCKF only supports a memory operand, so we have to use a temporary. + SDValue StackPtr = DAG.CreateStackTemporary(MVT::i64); + int SPFI = cast(StackPtr.getNode())->getIndex(); + MachinePointerInfo MPI = + MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI); + + // Use STCFK to store the TOD clock into the temporary. + SDValue StoreOps[] = {Chain, StackPtr}; + Chain = DAG.getMemIntrinsicNode( + SystemZISD::STCKF, DL, DAG.getVTList(MVT::Other), StoreOps, MVT::i64, + MPI, MaybeAlign(), MachineMemOperand::MOStore); + + // And read it back from there. + return DAG.getLoad(MVT::i64, DL, Chain, StackPtr, MPI); +} + SDValue SystemZTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { switch (Op.getOpcode()) { @@ -6199,6 +6223,8 @@ SDValue SystemZTargetLowering::LowerOperation(SDValue Op, return lowerIS_FPCLASS(Op, DAG); case ISD::GET_ROUNDING: return lowerGET_ROUNDING(Op, DAG); + case ISD::READCYCLECOUNTER: + return lowerREADCYCLECOUNTER(Op, DAG); default: llvm_unreachable("Unexpected node to lower"); } @@ -6425,6 +6451,7 @@ const char *SystemZTargetLowering::getTargetNodeName(unsigned Opcode) const { OPCODE(STRV); OPCODE(VLER); OPCODE(VSTER); + OPCODE(STCKF); OPCODE(PREFETCH); OPCODE(ADA_ENTRY); } @@ -6985,6 +7012,17 @@ SDValue SystemZTargetLowering::combineSTORE( } } + // Combine STORE (READCYCLECOUNTER) into STCKF. + if (!SN->isTruncatingStore() && + Op1.getOpcode() == ISD::READCYCLECOUNTER && + Op1.hasOneUse() && + N->getOperand(0).reachesChainWithoutSideEffects(SDValue(Op1.getNode(), 1))) { + SDValue Ops[] = { Op1.getOperand(0), N->getOperand(2) }; + return DAG.getMemIntrinsicNode(SystemZISD::STCKF, SDLoc(N), + DAG.getVTList(MVT::Other), + Ops, MemVT, SN->getMemOperand()); + } + // Transform a store of an i128 moved from GPRs into two separate stores. if (MemVT == MVT::i128 && SN->isSimple() && ISD::isNormalStore(SN)) { SDValue LoPart, HiPart; diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.h b/llvm/lib/Target/SystemZ/SystemZISelLowering.h index 406a13b9281c..4943c5cb703c 100644 --- a/llvm/lib/Target/SystemZ/SystemZISelLowering.h +++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.h @@ -378,6 +378,9 @@ enum NodeType : unsigned { // Element swapping load/store. Same operands as regular load/store. VLER, VSTER, + // Use STORE CLOCK FAST to store current TOD clock value. + STCKF, + // Prefetch from the second operand using the 4-bit control code in // the first operand. The code is 1 for a load prefetch and 2 for // a store prefetch. @@ -717,6 +720,7 @@ private: SDValue lowerShift(SDValue Op, SelectionDAG &DAG, unsigned ByScalar) const; SDValue lowerIS_FPCLASS(SDValue Op, SelectionDAG &DAG) const; SDValue lowerGET_ROUNDING(SDValue Op, SelectionDAG &DAG) const; + SDValue lowerREADCYCLECOUNTER(SDValue Op, SelectionDAG &DAG) const; bool canTreatAsByteVector(EVT VT) const; SDValue combineExtract(const SDLoc &DL, EVT ElemVT, EVT VecVT, SDValue OrigOp, diff --git a/llvm/lib/Target/SystemZ/SystemZInstrSystem.td b/llvm/lib/Target/SystemZ/SystemZInstrSystem.td index e26417ddb72d..497844bec85d 100644 --- a/llvm/lib/Target/SystemZ/SystemZInstrSystem.td +++ b/llvm/lib/Target/SystemZ/SystemZInstrSystem.td @@ -339,7 +339,7 @@ let hasSideEffects = 1 in // Store clock (fast / extended). let hasSideEffects = 1, Defs = [CC] in { def STCK : StoreInherentS<"stck", 0xB205, null_frag, 8>; - def STCKF : StoreInherentS<"stckf", 0xB27C, null_frag, 8>; + def STCKF : StoreInherentS<"stckf", 0xB27C, z_stckf, 8>; def STCKE : StoreInherentS<"stcke", 0xB278, null_frag, 16>; } diff --git a/llvm/lib/Target/SystemZ/SystemZOperators.td b/llvm/lib/Target/SystemZ/SystemZOperators.td index 6c4e33a6aa7f..1611436b01b7 100644 --- a/llvm/lib/Target/SystemZ/SystemZOperators.td +++ b/llvm/lib/Target/SystemZ/SystemZOperators.td @@ -132,6 +132,8 @@ def SDT_ZIPM : SDTypeProfile<1, 1, def SDT_ZPrefetch : SDTypeProfile<0, 2, [SDTCisVT<0, i32>, SDTCisPtrTy<1>]>; +def SDT_ZStoreInherent : SDTypeProfile<0, 1, + [SDTCisPtrTy<0>]>; def SDT_ZTBegin : SDTypeProfile<1, 2, [SDTCisVT<0, i32>, SDTCisPtrTy<1>, @@ -307,6 +309,8 @@ def z_loadeswap : SDNode<"SystemZISD::VLER", SDTLoad, [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>; def z_storeeswap : SDNode<"SystemZISD::VSTER", SDTStore, [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>; +def z_stckf : SDNode<"SystemZISD::STCKF", SDT_ZStoreInherent, + [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>; def z_tdc : SDNode<"SystemZISD::TDC", SDT_ZTest>; diff --git a/llvm/test/CodeGen/SystemZ/readcyclecounter.ll b/llvm/test/CodeGen/SystemZ/readcyclecounter.ll new file mode 100644 index 000000000000..34b6d34143f8 --- /dev/null +++ b/llvm/test/CodeGen/SystemZ/readcyclecounter.ll @@ -0,0 +1,27 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=s390x-ibm-linux | FileCheck %s + +; Verify that we correctly lower ISD::READCYCLECOUNTER. + +define i64 @test_builtin_readcyclecounter1() { +; CHECK-LABEL: test_builtin_readcyclecounter1: +; CHECK: # %bb.0: +; CHECK-NEXT: aghi %r15, -168 +; CHECK-NEXT: .cfi_def_cfa_offset 328 +; CHECK-NEXT: stckf 160(%r15) +; CHECK-NEXT: lg %r2, 160(%r15) +; CHECK-NEXT: aghi %r15, 168 +; CHECK-NEXT: br %r14 + %1 = tail call i64 @llvm.readcyclecounter() + ret i64 %1 +} + +define void @test_builtin_readcyclecounter2(ptr %ptr) { +; CHECK-LABEL: test_builtin_readcyclecounter2: +; CHECK: # %bb.0: +; CHECK-NEXT: stckf 0(%r2) +; CHECK-NEXT: br %r14 + %1 = tail call i64 @llvm.readcyclecounter() + store i64 %1, ptr %ptr + ret void +} -- GitLab From ee9fa38554f8427f5361708b28551e70ff0e1256 Mon Sep 17 00:00:00 2001 From: Schrodinger ZHU Yifan Date: Fri, 22 Mar 2024 15:20:00 -0400 Subject: [PATCH 098/306] [libc] fix missing macro dependency in bazel (#86298) ![image](https://github.com/llvm/llvm-project/assets/20108837/94cb5718-3526-4bae-8a79-f5b1d19b352d) --- .../llvm-project-overlay/libc/utils/MPFRWrapper/BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/bazel/llvm-project-overlay/libc/utils/MPFRWrapper/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/utils/MPFRWrapper/BUILD.bazel index 803010e8b3ad..5f59d70ecc16 100644 --- a/utils/bazel/llvm-project-overlay/libc/utils/MPFRWrapper/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/utils/MPFRWrapper/BUILD.bazel @@ -46,6 +46,7 @@ libc_support_library( "//libc:__support_cpp_type_traits", "//libc:__support_fputil_fp_bits", "//libc:__support_fputil_fpbits_str", + "//libc:llvm_libc_macros_math_macros", "//libc/test/UnitTest:LibcUnitTest", "//libc/test/UnitTest:fp_test_helpers", "//libc/utils/MPFRWrapper:mpfr_impl", -- GitLab From d394f3a162b871668d0c8e8bf6a94922fa8698ae Mon Sep 17 00:00:00 2001 From: Xing Xue Date: Fri, 22 Mar 2024 15:25:08 -0400 Subject: [PATCH 099/306] [OpenMP][AIX] Affinity implementation for AIX (#84984) This patch implements `affinity` for AIX, which is quite different from platforms such as Linux. - Setting CPU affinity through masks and related functions are not supported. System call `bindprocessor()` is used to bind a thread to one CPU per call. - There are no system routines to get the affinity info of a thread. The implementation of `get_system_affinity()` for AIX gets the mask of all available CPUs, to be used as the full mask only. - Topology is not available from the file system. It is obtained through system SRAD (Scheduler Resource Allocation Domain). This patch has run through the libomp LIT tests successfully with `affinity` enabled. --- openmp/runtime/src/kmp.h | 5 +- openmp/runtime/src/kmp_affinity.cpp | 130 ++++++++++++++++++++++++++-- openmp/runtime/src/kmp_affinity.h | 74 +++++++++++++++- openmp/runtime/src/kmp_os.h | 2 +- openmp/runtime/src/z_Linux_util.cpp | 39 +++++++-- openmp/runtime/test/lit.cfg | 2 +- 6 files changed, 233 insertions(+), 19 deletions(-) diff --git a/openmp/runtime/src/kmp.h b/openmp/runtime/src/kmp.h index 885d6636abe4..18ccf10fe17d 100644 --- a/openmp/runtime/src/kmp.h +++ b/openmp/runtime/src/kmp.h @@ -819,6 +819,7 @@ private: typedef KMPAffinity::Mask kmp_affin_mask_t; extern KMPAffinity *__kmp_affinity_dispatch; +#ifndef KMP_OS_AIX class kmp_affinity_raii_t { kmp_affin_mask_t *mask; bool restored; @@ -843,6 +844,7 @@ public: } ~kmp_affinity_raii_t() { restore(); } }; +#endif // !KMP_OS_AIX // Declare local char buffers with this size for printing debug and info // messages, using __kmp_affinity_print_mask(). @@ -3910,7 +3912,8 @@ extern void __kmp_balanced_affinity(kmp_info_t *th, int team_size); #if KMP_WEIGHTED_ITERATIONS_SUPPORTED extern int __kmp_get_first_osid_with_ecore(void); #endif -#if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY +#if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY || \ + KMP_OS_AIX extern int kmp_set_thread_affinity_mask_initial(void); #endif static inline void __kmp_assign_root_init_mask() { diff --git a/openmp/runtime/src/kmp_affinity.cpp b/openmp/runtime/src/kmp_affinity.cpp index 048bd174fc95..b574dbbaf54f 100644 --- a/openmp/runtime/src/kmp_affinity.cpp +++ b/openmp/runtime/src/kmp_affinity.cpp @@ -2910,12 +2910,17 @@ static inline const char *__kmp_cpuinfo_get_envvar() { } // Parse /proc/cpuinfo (or an alternate file in the same format) to obtain the -// affinity map. +// affinity map. On AIX, the map is obtained through system SRAD (Scheduler +// Resource Allocation Domain). static bool __kmp_affinity_create_cpuinfo_map(int *line, kmp_i18n_id_t *const msg_id) { + *msg_id = kmp_i18n_null; + +#if KMP_OS_AIX + unsigned num_records = __kmp_xproc; +#else const char *filename = __kmp_cpuinfo_get_filename(); const char *envvar = __kmp_cpuinfo_get_envvar(); - *msg_id = kmp_i18n_null; if (__kmp_affinity.flags.verbose) { KMP_INFORM(AffParseFilename, "KMP_AFFINITY", filename); @@ -2974,6 +2979,7 @@ static bool __kmp_affinity_create_cpuinfo_map(int *line, *msg_id = kmp_i18n_str_CantRewindCpuinfo; return false; } +#endif // KMP_OS_AIX // Allocate the array of records to store the proc info in. The dummy // element at the end makes the logic in filling them out easier to code. @@ -3003,6 +3009,99 @@ static bool __kmp_affinity_create_cpuinfo_map(int *line, INIT_PROC_INFO(threadInfo[i]); } +#if KMP_OS_AIX + int smt_threads; + lpar_info_format1_t cpuinfo; + unsigned num_avail = __kmp_xproc; + + if (__kmp_affinity.flags.verbose) + KMP_INFORM(AffParseFilename, "KMP_AFFINITY", "system info for topology"); + + // Get the number of SMT threads per core. + int retval = + lpar_get_info(LPAR_INFO_FORMAT1, &cpuinfo, sizeof(lpar_info_format1_t)); + if (!retval) + smt_threads = cpuinfo.smt_threads; + else { + CLEANUP_THREAD_INFO; + *msg_id = kmp_i18n_str_UnknownTopology; + return false; + } + + // Allocate a resource set containing available system resourses. + rsethandle_t sys_rset = rs_alloc(RS_SYSTEM); + if (sys_rset == NULL) { + CLEANUP_THREAD_INFO; + *msg_id = kmp_i18n_str_UnknownTopology; + return false; + } + // Allocate a resource set for the SRAD info. + rsethandle_t srad = rs_alloc(RS_EMPTY); + if (srad == NULL) { + rs_free(sys_rset); + CLEANUP_THREAD_INFO; + *msg_id = kmp_i18n_str_UnknownTopology; + return false; + } + + // Get the SRAD system detail level. + int sradsdl = rs_getinfo(NULL, R_SRADSDL, 0); + if (sradsdl < 0) { + rs_free(sys_rset); + rs_free(srad); + CLEANUP_THREAD_INFO; + *msg_id = kmp_i18n_str_UnknownTopology; + return false; + } + // Get the number of RADs at that SRAD SDL. + int num_rads = rs_numrads(sys_rset, sradsdl, 0); + if (num_rads < 0) { + rs_free(sys_rset); + rs_free(srad); + CLEANUP_THREAD_INFO; + *msg_id = kmp_i18n_str_UnknownTopology; + return false; + } + + // Get the maximum number of procs that may be contained in a resource set. + int max_procs = rs_getinfo(NULL, R_MAXPROCS, 0); + if (max_procs < 0) { + rs_free(sys_rset); + rs_free(srad); + CLEANUP_THREAD_INFO; + *msg_id = kmp_i18n_str_UnknownTopology; + return false; + } + + int cur_rad = 0; + int num_set = 0; + for (int srad_idx = 0; cur_rad < num_rads && srad_idx < VMI_MAXRADS; + ++srad_idx) { + // Check if the SRAD is available in the RSET. + if (rs_getrad(sys_rset, srad, sradsdl, srad_idx, 0) < 0) + continue; + + for (int cpu = 0; cpu < max_procs; cpu++) { + // Set the info for the cpu if it is in the SRAD. + if (rs_op(RS_TESTRESOURCE, srad, NULL, R_PROCS, cpu)) { + threadInfo[cpu][osIdIndex] = cpu; + threadInfo[cpu][pkgIdIndex] = cur_rad; + threadInfo[cpu][coreIdIndex] = cpu / smt_threads; + ++num_set; + if (num_set >= num_avail) { + // Done if all available CPUs have been set. + break; + } + } + } + ++cur_rad; + } + rs_free(sys_rset); + rs_free(srad); + + // The topology is already sorted. + +#else // !KMP_OS_AIX unsigned num_avail = 0; *line = 0; #if KMP_ARCH_S390X @@ -3250,6 +3349,8 @@ static bool __kmp_affinity_create_cpuinfo_map(int *line, qsort(threadInfo, num_avail, sizeof(*threadInfo), __kmp_affinity_cmp_ProcCpuInfo_phys_id); +#endif // KMP_OS_AIX + // The table is now sorted by pkgId / coreId / threadId, but we really don't // know the radix of any of the fields. pkgId's may be sparsely assigned among // the chips on a system. Although coreId's are usually assigned @@ -4445,7 +4546,7 @@ static bool __kmp_aux_affinity_initialize_topology(kmp_affinity_t &affinity) { } #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */ -#if KMP_OS_LINUX +#if KMP_OS_LINUX || KMP_OS_AIX if (!success) { int line = 0; success = __kmp_affinity_create_cpuinfo_map(&line, &msg_id); @@ -4841,7 +4942,12 @@ void __kmp_affinity_uninitialize(void) { } if (__kmp_affin_origMask != NULL) { if (KMP_AFFINITY_CAPABLE()) { +#if KMP_OS_AIX + // Uninitialize by unbinding the thread. + bindprocessor(BINDTHREAD, thread_self(), PROCESSOR_CLASS_ANY); +#else __kmp_set_system_affinity(__kmp_affin_origMask, FALSE); +#endif } KMP_CPU_FREE(__kmp_affin_origMask); __kmp_affin_origMask = NULL; @@ -5015,7 +5121,10 @@ void __kmp_affinity_bind_init_mask(int gtid) { __kmp_set_system_affinity(th->th.th_affin_mask, FALSE); } else #endif +#ifndef KMP_OS_AIX + // Do not set the full mask as the init mask on AIX. __kmp_set_system_affinity(th->th.th_affin_mask, TRUE); +#endif } void __kmp_affinity_bind_place(int gtid) { @@ -5128,7 +5237,7 @@ int __kmp_aux_set_affinity(void **mask) { int __kmp_aux_get_affinity(void **mask) { int gtid; int retval; -#if KMP_OS_WINDOWS || KMP_DEBUG +#if KMP_OS_WINDOWS || KMP_OS_AIX || KMP_DEBUG kmp_info_t *th; #endif if (!KMP_AFFINITY_CAPABLE()) { @@ -5136,7 +5245,7 @@ int __kmp_aux_get_affinity(void **mask) { } gtid = __kmp_entry_gtid(); -#if KMP_OS_WINDOWS || KMP_DEBUG +#if KMP_OS_WINDOWS || KMP_OS_AIX || KMP_DEBUG th = __kmp_threads[gtid]; #else (void)gtid; // unused variable @@ -5159,7 +5268,7 @@ int __kmp_aux_get_affinity(void **mask) { } } -#if !KMP_OS_WINDOWS +#if !KMP_OS_WINDOWS && !KMP_OS_AIX retval = __kmp_get_system_affinity((kmp_affin_mask_t *)(*mask), FALSE); KA_TRACE( @@ -5179,7 +5288,7 @@ int __kmp_aux_get_affinity(void **mask) { KMP_CPU_COPY((kmp_affin_mask_t *)(*mask), th->th.th_affin_mask); return 0; -#endif /* KMP_OS_WINDOWS */ +#endif /* !KMP_OS_WINDOWS && !KMP_OS_AIX */ } int __kmp_aux_get_affinity_max_proc() { @@ -5561,7 +5670,8 @@ void __kmp_balanced_affinity(kmp_info_t *th, int nthreads) { } } -#if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY +#if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY || \ + KMP_OS_AIX // We don't need this entry for Windows because // there is GetProcessAffinityMask() api // @@ -5596,7 +5706,11 @@ extern "C" "set full mask for thread %d\n", gtid)); KMP_DEBUG_ASSERT(__kmp_affin_fullMask != NULL); +#if KMP_OS_AIX + return bindprocessor(BINDTHREAD, thread_self(), PROCESSOR_CLASS_ANY); +#else return __kmp_set_system_affinity(__kmp_affin_fullMask, FALSE); +#endif } #endif diff --git a/openmp/runtime/src/kmp_affinity.h b/openmp/runtime/src/kmp_affinity.h index 1c7db2f59943..7efc090f8863 100644 --- a/openmp/runtime/src/kmp_affinity.h +++ b/openmp/runtime/src/kmp_affinity.h @@ -191,7 +191,8 @@ public: }; #endif /* KMP_USE_HWLOC */ -#if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY +#if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY || \ + KMP_OS_AIX #if KMP_OS_LINUX /* On some of the older OS's that we build on, these constants aren't present in #included from . They must be the same on @@ -317,6 +318,10 @@ public: #elif KMP_OS_NETBSD #include #include +#elif KMP_OS_AIX +#include +#include +#define VMI_MAXRADS 64 // Maximum number of RADs allowed by AIX. #endif class KMPNativeAffinity : public KMPAffinity { class Mask : public KMPAffinity::Mask { @@ -404,6 +409,70 @@ class KMPNativeAffinity : public KMPAffinity { ++retval; return retval; } +#if KMP_OS_AIX + // On AIX, we don't have a way to get CPU(s) a thread is bound to. + // This routine is only used to get the full mask. + int get_system_affinity(bool abort_on_error) override { + KMP_ASSERT2(KMP_AFFINITY_CAPABLE(), + "Illegal get affinity operation when not capable"); + + (void)abort_on_error; + + // Set the mask with all CPUs that are available. + for (int i = 0; i < __kmp_xproc; ++i) + KMP_CPU_SET(i, this); + return 0; + } + int set_system_affinity(bool abort_on_error) const override { + KMP_ASSERT2(KMP_AFFINITY_CAPABLE(), + + "Illegal set affinity operation when not capable"); + + int location; + int gtid = __kmp_entry_gtid(); + int tid = thread_self(); + + // Unbind the thread if it was bound to any processors before so that + // we can bind the thread to CPUs specified by the mask not others. + int retval = bindprocessor(BINDTHREAD, tid, PROCESSOR_CLASS_ANY); + + // On AIX, we can only bind to one instead of a set of CPUs with the + // bindprocessor() system call. + KMP_CPU_SET_ITERATE(location, this) { + if (KMP_CPU_ISSET(location, this)) { + retval = bindprocessor(BINDTHREAD, tid, location); + if (retval == -1 && errno == 1) { + rsid_t rsid; + rsethandle_t rsh; + // Put something in rsh to prevent compiler warning + // about uninitalized use + rsh = rs_alloc(RS_EMPTY); + rsid.at_pid = getpid(); + if (RS_DEFAULT_RSET != ra_getrset(R_PROCESS, rsid, 0, rsh)) { + retval = ra_detachrset(R_PROCESS, rsid, 0); + retval = bindprocessor(BINDTHREAD, tid, location); + } + } + if (retval == 0) { + KA_TRACE(10, ("__kmp_set_system_affinity: Done binding " + "T#%d to cpu=%d.\n", + gtid, location)); + continue; + } + int error = errno; + if (abort_on_error) { + __kmp_fatal(KMP_MSG(FunctionError, "bindprocessor()"), + KMP_ERR(error), __kmp_msg_null); + KA_TRACE(10, ("__kmp_set_system_affinity: Error binding " + "T#%d to cpu=%d, errno=%d.\n", + gtid, location, error)); + return error; + } + } + } + return 0; + } +#else // !KMP_OS_AIX int get_system_affinity(bool abort_on_error) override { KMP_ASSERT2(KMP_AFFINITY_CAPABLE(), "Illegal get affinity operation when not capable"); @@ -446,6 +515,7 @@ class KMPNativeAffinity : public KMPAffinity { } return error; } +#endif // KMP_OS_AIX }; void determine_capable(const char *env_var) override { __kmp_affinity_determine_capable(env_var); @@ -475,7 +545,7 @@ class KMPNativeAffinity : public KMPAffinity { api_type get_api_type() const override { return NATIVE_OS; } }; #endif /* KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY \ - */ + || KMP_OS_AIX */ #if KMP_OS_WINDOWS class KMPNativeAffinity : public KMPAffinity { diff --git a/openmp/runtime/src/kmp_os.h b/openmp/runtime/src/kmp_os.h index 63da9e5fa15d..a628070c882a 100644 --- a/openmp/runtime/src/kmp_os.h +++ b/openmp/runtime/src/kmp_os.h @@ -76,7 +76,7 @@ #endif #if (KMP_OS_LINUX || KMP_OS_WINDOWS || KMP_OS_FREEBSD || KMP_OS_NETBSD || \ - KMP_OS_DRAGONFLY) && \ + KMP_OS_DRAGONFLY || KMP_OS_AIX) && \ !KMP_OS_WASI #define KMP_AFFINITY_SUPPORTED 1 #if KMP_OS_WINDOWS && KMP_ARCH_X86_64 diff --git a/openmp/runtime/src/z_Linux_util.cpp b/openmp/runtime/src/z_Linux_util.cpp index d751a417331c..29db9d008a49 100644 --- a/openmp/runtime/src/z_Linux_util.cpp +++ b/openmp/runtime/src/z_Linux_util.cpp @@ -125,7 +125,8 @@ static void __kmp_print_cond(char *buffer, kmp_cond_align_t *cond) { } #endif -#if ((KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY) && \ +#if ((KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY || \ + KMP_OS_AIX) && \ KMP_AFFINITY_SUPPORTED) /* Affinity support */ @@ -142,6 +143,29 @@ void __kmp_affinity_bind_thread(int which) { KMP_CPU_FREE_FROM_STACK(mask); } +#if KMP_OS_AIX +void __kmp_affinity_determine_capable(const char *env_var) { + // All versions of AIX support bindprocessor(). + + size_t mask_size = __kmp_xproc / CHAR_BIT; + // Round up to byte boundary. + if (__kmp_xproc % CHAR_BIT) + ++mask_size; + + // Round up to the mask_size_type boundary. + if (mask_size % sizeof(__kmp_affin_mask_size)) + mask_size += sizeof(__kmp_affin_mask_size) - + mask_size % sizeof(__kmp_affin_mask_size); + KMP_AFFINITY_ENABLE(mask_size); + KA_TRACE(10, + ("__kmp_affinity_determine_capable: " + "AIX OS affinity interface bindprocessor functional (mask size = " + "%" KMP_SIZE_T_SPEC ").\n", + __kmp_affin_mask_size)); +} + +#else // !KMP_OS_AIX + /* Determine if we can access affinity functionality on this version of * Linux* OS by checking __NR_sched_{get,set}affinity system calls, and set * __kmp_affin_mask_size to the appropriate value (0 means not capable). */ @@ -271,8 +295,9 @@ void __kmp_affinity_determine_capable(const char *env_var) { KMP_WARNING(AffCantGetMaskSize, env_var); } } - -#endif // KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED +#endif // KMP_OS_AIX +#endif // (KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || \ + KMP_OS_DRAGONFLY || KMP_OS_AIX) && KMP_AFFINITY_SUPPORTED #if KMP_USE_FUTEX @@ -501,7 +526,7 @@ static void *__kmp_launch_worker(void *thr) { #endif /* KMP_BLOCK_SIGNALS */ void *exit_val; #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \ - KMP_OS_OPENBSD || KMP_OS_HURD || KMP_OS_SOLARIS + KMP_OS_OPENBSD || KMP_OS_HURD || KMP_OS_SOLARIS || KMP_OS_AIX void *volatile padding = 0; #endif int gtid; @@ -550,7 +575,7 @@ static void *__kmp_launch_worker(void *thr) { #endif /* KMP_BLOCK_SIGNALS */ #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \ - KMP_OS_OPENBSD || KMP_OS_HURD || KMP_OS_SOLARIS + KMP_OS_OPENBSD || KMP_OS_HURD || KMP_OS_SOLARIS || KMP_OS_AIX if (__kmp_stkoffset > 0 && gtid > 0) { padding = KMP_ALLOCA(gtid * __kmp_stkoffset); (void)padding; @@ -1268,7 +1293,8 @@ static void __kmp_atfork_child(void) { ++__kmp_fork_count; #if KMP_AFFINITY_SUPPORTED -#if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY +#if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY || \ + KMP_OS_AIX // reset the affinity in the child to the initial thread // affinity in the parent kmp_set_thread_affinity_mask_initial(); @@ -2325,6 +2351,7 @@ int __kmp_is_address_mapped(void *addr) { found = (int)addr < (__builtin_wasm_memory_size(0) * PAGESIZE); #elif KMP_OS_AIX + (void)rc; // FIXME(AIX): Implement this found = 1; diff --git a/openmp/runtime/test/lit.cfg b/openmp/runtime/test/lit.cfg index a3456063c10f..e27e52bb4289 100644 --- a/openmp/runtime/test/lit.cfg +++ b/openmp/runtime/test/lit.cfg @@ -129,7 +129,7 @@ if config.operating_system == 'NetBSD': if config.operating_system == 'Darwin': config.available_features.add("darwin") -if config.operating_system in ['Windows', 'Linux', 'FreeBSD', 'NetBSD', 'DragonFly']: +if config.operating_system in ['Windows', 'Linux', 'FreeBSD', 'NetBSD', 'DragonFly', 'AIX']: config.available_features.add('affinity') if config.operating_system in ['Linux']: -- GitLab From dfe4ca9b7f4a422500d78280dc5eefd1979939e6 Mon Sep 17 00:00:00 2001 From: Paul Kirth Date: Fri, 22 Mar 2024 12:27:41 -0700 Subject: [PATCH 100/306] [RISCV][lld] Set the type of TLSDESC relocation's referenced local symbol to STT_NOTYPE When adding fixups for RISCV_TLSDESC_ADD_LO and RISCV_TLSDESC_LOAD_LO, the local label added for RISCV TLSDESC relocations have STT_TLS set, which is incorrect. Instead, these labels should have `STT_NOTYPE`. This patch stops adding such fixups and avoid setting the STT_TLS on these symbols. Failing to do so can cause LLD to emit an error `has an STT_TLS symbol but doesn't have an SHF_TLS section`. We additionally, adjust how LLD services these relocations to avoid errors with incompatible relocation and symbol types. Reviewers: topperc, MaskRay Reviewed By: MaskRay Pull Request: https://github.com/llvm/llvm-project/pull/85817 --- lld/ELF/Relocations.cpp | 5 +++- lld/test/ELF/riscv-tlsdesc-relax.s | 8 ++++++ lld/test/ELF/riscv-tlsdesc.s | 27 +++++++++++-------- .../Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp | 2 -- 4 files changed, 28 insertions(+), 14 deletions(-) diff --git a/lld/ELF/Relocations.cpp b/lld/ELF/Relocations.cpp index 619fbaf5dc54..92a1b9baaca3 100644 --- a/lld/ELF/Relocations.cpp +++ b/lld/ELF/Relocations.cpp @@ -1480,7 +1480,10 @@ template void RelocationScanner::scanOne(RelTy *&i) { // Process TLS relocations, including TLS optimizations. Note that // R_TPREL and R_TPREL_NEG relocations are resolved in processAux. - if (sym.isTls()) { + // + // Some RISCV TLSDESC relocations reference a local NOTYPE symbol, + // but we need to process them in handleTlsRelocation. + if (sym.isTls() || oneof(expr)) { if (unsigned processed = handleTlsRelocation(type, sym, *sec, offset, addend, expr)) { i += processed - 1; diff --git a/lld/test/ELF/riscv-tlsdesc-relax.s b/lld/test/ELF/riscv-tlsdesc-relax.s index fb24317e6535..5718d4175be1 100644 --- a/lld/test/ELF/riscv-tlsdesc-relax.s +++ b/lld/test/ELF/riscv-tlsdesc-relax.s @@ -33,12 +33,14 @@ # GD64-NEXT: c.add a0, tp # GD64-NEXT: jal {{.*}} ## &.got[c]-. = 0x20c0+8 - 0x1020 = 0x10a8 +# GD64-LABEL: <.Ltlsdesc_hi1>: # GD64-NEXT: 1020: auipc a4, 0x1 # GD64-NEXT: ld a5, 0xa8(a4) # GD64-NEXT: addi a0, a4, 0xa8 # GD64-NEXT: jalr t0, 0x0(a5) # GD64-NEXT: c.add a0, tp ## &.got[c]-. = 0x20c0+8 - 0x1032 = 0x1096 +# GD64-LABEL: <.Ltlsdesc_hi2>: # GD64-NEXT: 1032: auipc a6, 0x1 # GD64-NEXT: ld a7, 0x96(a6) # GD64-NEXT: addi a0, a6, 0x96 @@ -64,6 +66,7 @@ # LE64-NEXT: jal {{.*}} # LE64-NEXT: R_RISCV_JAL foo # LE64-NEXT: R_RISCV_RELAX *ABS* +# LE64-LABEL: <.Ltlsdesc_hi1>: # LE64-NEXT: addi a0, zero, 0x7ff # LE64-NEXT: R_RISCV_TLSDESC_HI20 b # LE64-NEXT: R_RISCV_RELAX *ABS* @@ -71,6 +74,7 @@ # LE64-NEXT: R_RISCV_TLSDESC_ADD_LO12 .Ltlsdesc_hi1 # LE64-NEXT: R_RISCV_TLSDESC_CALL .Ltlsdesc_hi1 # LE64-NEXT: c.add a0, tp +# LE64-LABEL: <.Ltlsdesc_hi2>: # LE64-NEXT: addi zero, zero, 0x0 # LE64-NEXT: R_RISCV_TLSDESC_HI20 b # LE64-NEXT: addi zero, zero, 0x0 @@ -93,9 +97,11 @@ # LE64A-NEXT: addi a0, a0, -0x479 # LE64A-NEXT: c.add a0, tp # LE64A-NEXT: jal {{.*}} +# LE64A-LABEL: <.Ltlsdesc_hi1>: # LE64A-NEXT: lui a0, 0x2 # LE64A-NEXT: addi a0, a0, -0x479 # LE64A-NEXT: c.add a0, tp +# LE64A-LABEL: <.Ltlsdesc_hi2>: # LE64A-NEXT: addi zero, zero, 0x0 # LE64A-NEXT: addi zero, zero, 0x0 # LE64A-NEXT: lui a0, 0x2 @@ -115,10 +121,12 @@ # IE64-NEXT: c.add a0, tp # IE64-NEXT: jal {{.*}} ## &.got[c]-. = 0x120e0+8 - 0x11018 = 0x10d0 +# IE64-LABEL: <.Ltlsdesc_hi1>: # IE64-NEXT: 11018: auipc a0, 0x1 # IE64-NEXT: ld a0, 0xd0(a0) # IE64-NEXT: c.add a0, tp ## &.got[c]-. = 0x120e0+8 - 0x1102a = 0x10be +# IE64-LABEL: <.Ltlsdesc_hi2>: # IE64-NEXT: addi zero, zero, 0x0 # IE64-NEXT: addi zero, zero, 0x0 # IE64-NEXT: 1102a: auipc a0, 0x1 diff --git a/lld/test/ELF/riscv-tlsdesc.s b/lld/test/ELF/riscv-tlsdesc.s index c583e15cf30c..935ecbddfbff 100644 --- a/lld/test/ELF/riscv-tlsdesc.s +++ b/lld/test/ELF/riscv-tlsdesc.s @@ -29,11 +29,13 @@ # RUN: ld.lld -e 0 -z now a.32.o c.32.so -o a.32.ie # RUN: llvm-objdump --no-show-raw-insn -M no-aliases -h -d a.32.ie | FileCheck %s --check-prefix=IE32 -# RUN: llvm-mc -triple=riscv64 -filetype=obj d.s -o d.64.o -# RUN: not ld.lld -shared -soname=d.64.so -o d.64.so d.64.o 2>&1 | FileCheck %s --check-prefix=BADTLSLABEL +## Prior to https://github.com/llvm/llvm-project/pull/85817 the local TLSDESC +## labels would be marked STT_TLS, resulting in an error "has an STT_TLS symbol but doesn't have an SHF_TLS section" +# RUN: llvm-mc -triple=riscv64 -filetype=obj d.s -o d.64.o +# RUN: ld.lld -shared -soname=d.64.so -o d.64.so d.64.o --fatal-warnings # RUN: llvm-mc -triple=riscv32 -filetype=obj d.s -o d.32.o --defsym ELF32=1 -# RUN: not ld.lld -shared -soname=d.32.so -o d.32.so d.32.o 2>&1 | FileCheck %s --check-prefix=BADTLSLABEL +# RUN: ld.lld -shared -soname=d.32.so -o d.32.so d.32.o --fatal-warnings # GD64-RELA: .rela.dyn { # GD64-RELA-NEXT: 0x2408 R_RISCV_TLSDESC - 0x7FF @@ -74,14 +76,14 @@ # GD64-NEXT: add a0, a0, tp ## &.got[b]-. = 0x23e0+40 - 0x12f4 = 0x1114 -# GD64-NEXT: 12f4: auipc a2, 0x1 +# GD64: 12f4: auipc a2, 0x1 # GD64-NEXT: ld a3, 0x114(a2) # GD64-NEXT: addi a0, a2, 0x114 # GD64-NEXT: jalr t0, 0x0(a3) # GD64-NEXT: add a0, a0, tp ## &.got[c]-. = 0x23e0+24 - 0x1308 = 0x10f0 -# GD64-NEXT: 1308: auipc a4, 0x1 +# GD64: 1308: auipc a4, 0x1 # GD64-NEXT: ld a5, 0xf0(a4) # GD64-NEXT: addi a0, a4, 0xf0 # GD64-NEXT: jalr t0, 0x0(a5) @@ -89,7 +91,7 @@ # NOREL: no relocations -# LE64-LABEL: <.text>: +# LE64-LABEL: <.Ltlsdesc_hi0>: ## st_value(a) = 8 # LE64-NEXT: addi zero, zero, 0x0 # LE64-NEXT: addi zero, zero, 0x0 @@ -97,12 +99,14 @@ # LE64-NEXT: addi a0, zero, 0x8 # LE64-NEXT: add a0, a0, tp ## st_value(b) = 2047 +# LE64-LABEL: <.Ltlsdesc_hi1>: # LE64-NEXT: addi zero, zero, 0x0 # LE64-NEXT: addi zero, zero, 0x0 # LE64-NEXT: addi zero, zero, 0x0 # LE64-NEXT: addi a0, zero, 0x7ff # LE64-NEXT: add a0, a0, tp ## st_value(c) = 2048 +# LE64-LABEL: <.Ltlsdesc_hi2>: # LE64-NEXT: addi zero, zero, 0x0 # LE64-NEXT: addi zero, zero, 0x0 # LE64-NEXT: lui a0, 0x1 @@ -116,18 +120,20 @@ # IE64: .got 00000010 00000000000123a8 ## a and b are optimized to use LE. c is optimized to IE. -# IE64-LABEL: <.text>: +# IE64-LABEL: <.Ltlsdesc_hi0>: # IE64-NEXT: addi zero, zero, 0x0 # IE64-NEXT: addi zero, zero, 0x0 # IE64-NEXT: addi zero, zero, 0x0 # IE64-NEXT: addi a0, zero, 0x8 # IE64-NEXT: add a0, a0, tp +# IE64-LABEL: <.Ltlsdesc_hi1>: # IE64-NEXT: addi zero, zero, 0x0 # IE64-NEXT: addi zero, zero, 0x0 # IE64-NEXT: addi zero, zero, 0x0 # IE64-NEXT: addi a0, zero, 0x7ff # IE64-NEXT: add a0, a0, tp ## &.got[c]-. = 0x123a8+8 - 0x112b8 = 0x10f8 +# IE64-LABEL: <.Ltlsdesc_hi2>: # IE64-NEXT: addi zero, zero, 0x0 # IE64-NEXT: addi zero, zero, 0x0 # IE64-NEXT: 112b8: auipc a0, 0x1 @@ -136,7 +142,7 @@ # IE32: .got 00000008 00012248 -# IE32-LABEL: <.text>: +# IE32-LABEL: <.Ltlsdesc_hi0>: ## st_value(a) = 8 # IE32-NEXT: addi zero, zero, 0x0 # IE32-NEXT: addi zero, zero, 0x0 @@ -144,21 +150,20 @@ # IE32-NEXT: addi a0, zero, 0x8 # IE32-NEXT: add a0, a0, tp ## st_value(b) = 2047 +# IE32-LABEL: <.Ltlsdesc_hi1>: # IE32-NEXT: addi zero, zero, 0x0 # IE32-NEXT: addi zero, zero, 0x0 # IE32-NEXT: addi zero, zero, 0x0 # IE32-NEXT: addi a0, zero, 0x7ff # IE32-NEXT: add a0, a0, tp ## &.got[c]-. = 0x12248+4 - 0x111cc = 0x1080 +# IE32-LABEL: <.Ltlsdesc_hi2>: # IE32-NEXT: addi zero, zero, 0x0 # IE32-NEXT: addi zero, zero, 0x0 # IE32-NEXT: 111cc: auipc a0, 0x1 # IE32-NEXT: lw a0, 0x80(a0) # IE32-NEXT: add a0, a0, tp -## FIXME This should not pass, but the code MC layer needs a fix to prevent this. -# BADTLSLABEL: error: d.{{.*}}.o has an STT_TLS symbol but doesn't have an SHF_TLS section - #--- a.s .macro load dst, src .ifdef ELF32 diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp index 254a9a4bc0ef..b8e0f3a867f4 100644 --- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp +++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp @@ -207,8 +207,6 @@ void RISCVMCExpr::fixELFSymbolsInTLSFixups(MCAssembler &Asm) const { case VK_RISCV_TLS_GOT_HI: case VK_RISCV_TLS_GD_HI: case VK_RISCV_TLSDESC_HI: - case VK_RISCV_TLSDESC_ADD_LO: - case VK_RISCV_TLSDESC_LOAD_LO: break; } -- GitLab From e9639e9c0636d9e2b9591c2cdac5cac75e363e77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?= =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?= =?UTF-8?q?=E3=83=B3=29?= Date: Fri, 22 Mar 2024 12:56:45 -0700 Subject: [PATCH 101/306] [flang][NFC] Extract FIROpConversion to its own files (#86213) This PR extracts `FIROpConversion` and `FIROpAndTypeConversion` templated base patterns to a header file. All the functions from FIROpConversion that do not require the template argument are moved to a base class named `ConvertFIRToLLVMPattern`. This move is done so the `FIROpConversion` pattern and all its utility functions can be reused outside of the codegen pass. For the most part the code is only moved to the new files and not modified. The only update is that addition of the PatternBenefit argument with a default value to the constructor so it can be forwarded to the `ConversionPattern` ctor. This split is done in a similar way for the `ConvertOpToLLVMPattern` base pattern that is based on the `ConvertToLLVMPattern` base class in `mlir/include/mlir/Conversion/LLVMCommon/Pattern.h`. --- .../flang/Optimizer/CodeGen/FIROpPatterns.h | 248 +++++++++ .../flang/Optimizer/CodeGen/TypeConverter.h | 4 +- flang/lib/Optimizer/CodeGen/CMakeLists.txt | 1 + flang/lib/Optimizer/CodeGen/CodeGen.cpp | 486 +++--------------- flang/lib/Optimizer/CodeGen/FIROpPatterns.cpp | 315 ++++++++++++ 5 files changed, 636 insertions(+), 418 deletions(-) create mode 100644 flang/include/flang/Optimizer/CodeGen/FIROpPatterns.h create mode 100644 flang/lib/Optimizer/CodeGen/FIROpPatterns.cpp diff --git a/flang/include/flang/Optimizer/CodeGen/FIROpPatterns.h b/flang/include/flang/Optimizer/CodeGen/FIROpPatterns.h new file mode 100644 index 000000000000..06a44f188565 --- /dev/null +++ b/flang/include/flang/Optimizer/CodeGen/FIROpPatterns.h @@ -0,0 +1,248 @@ +//===-- FIROpPatterns.h -- FIR operation conversion patterns ----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef FORTRAN_OPTIMIZER_CODEGEN_FIROPPATTERNS_H +#define FORTRAN_OPTIMIZER_CODEGEN_FIROPPATTERNS_H + +#include "flang/Optimizer/CodeGen/TypeConverter.h" +#include "mlir/Conversion/LLVMCommon/Pattern.h" +#include "mlir/Dialect/LLVMIR/LLVMDialect.h" + +namespace fir { + +struct FIRToLLVMPassOptions; + +static constexpr unsigned defaultAddressSpace = 0u; + +class ConvertFIRToLLVMPattern : public mlir::ConvertToLLVMPattern { +public: + ConvertFIRToLLVMPattern(llvm::StringRef rootOpName, + mlir::MLIRContext *context, + const fir::LLVMTypeConverter &typeConverter, + const fir::FIRToLLVMPassOptions &options, + mlir::PatternBenefit benefit = 1); + +protected: + mlir::Type convertType(mlir::Type ty) const { + return lowerTy().convertType(ty); + } + + // Convert FIR type to LLVM without turning fir.box into memory + // reference. + mlir::Type convertObjectType(mlir::Type firType) const; + + mlir::LLVM::ConstantOp + genI32Constant(mlir::Location loc, mlir::ConversionPatternRewriter &rewriter, + int value) const; + + mlir::LLVM::ConstantOp + genConstantOffset(mlir::Location loc, + mlir::ConversionPatternRewriter &rewriter, + int offset) const; + + /// Perform an extension or truncation as needed on an integer value. Lowering + /// to the specific target may involve some sign-extending or truncation of + /// values, particularly to fit them from abstract box types to the + /// appropriate reified structures. + mlir::Value integerCast(mlir::Location loc, + mlir::ConversionPatternRewriter &rewriter, + mlir::Type ty, mlir::Value val) const; + struct TypePair { + mlir::Type fir; + mlir::Type llvm; + }; + + TypePair getBoxTypePair(mlir::Type firBoxTy) const; + + /// Construct code sequence to extract the specific value from a `fir.box`. + mlir::Value getValueFromBox(mlir::Location loc, TypePair boxTy, + mlir::Value box, mlir::Type resultTy, + mlir::ConversionPatternRewriter &rewriter, + int boxValue) const; + + /// Method to construct code sequence to get the triple for dimension `dim` + /// from a box. + llvm::SmallVector + getDimsFromBox(mlir::Location loc, llvm::ArrayRef retTys, + TypePair boxTy, mlir::Value box, mlir::Value dim, + mlir::ConversionPatternRewriter &rewriter) const; + + llvm::SmallVector + getDimsFromBox(mlir::Location loc, llvm::ArrayRef retTys, + TypePair boxTy, mlir::Value box, int dim, + mlir::ConversionPatternRewriter &rewriter) const; + + mlir::Value + loadDimFieldFromBox(mlir::Location loc, TypePair boxTy, mlir::Value box, + mlir::Value dim, int off, mlir::Type ty, + mlir::ConversionPatternRewriter &rewriter) const; + + mlir::Value + getDimFieldFromBox(mlir::Location loc, TypePair boxTy, mlir::Value box, + int dim, int off, mlir::Type ty, + mlir::ConversionPatternRewriter &rewriter) const; + + mlir::Value getStrideFromBox(mlir::Location loc, TypePair boxTy, + mlir::Value box, unsigned dim, + mlir::ConversionPatternRewriter &rewriter) const; + + /// Read base address from a fir.box. Returned address has type ty. + mlir::Value + getBaseAddrFromBox(mlir::Location loc, TypePair boxTy, mlir::Value box, + mlir::ConversionPatternRewriter &rewriter) const; + + mlir::Value + getElementSizeFromBox(mlir::Location loc, mlir::Type resultTy, TypePair boxTy, + mlir::Value box, + mlir::ConversionPatternRewriter &rewriter) const; + + // Get the element type given an LLVM type that is of the form + // (array|struct|vector)+ and the provided indexes. + mlir::Type getBoxEleTy(mlir::Type type, + llvm::ArrayRef indexes) const; + + // Return LLVM type of the object described by a fir.box of \p boxType. + mlir::Type getLlvmObjectTypeFromBoxType(mlir::Type boxType) const; + + /// Read the address of the type descriptor from a box. + mlir::Value + loadTypeDescAddress(mlir::Location loc, TypePair boxTy, mlir::Value box, + mlir::ConversionPatternRewriter &rewriter) const; + + // Load the attribute from the \p box and perform a check against \p maskValue + // The final comparison is implemented as `(attribute & maskValue) != 0`. + mlir::Value genBoxAttributeCheck(mlir::Location loc, TypePair boxTy, + mlir::Value box, + mlir::ConversionPatternRewriter &rewriter, + unsigned maskValue) const; + + template + mlir::LLVM::GEPOp genGEP(mlir::Location loc, mlir::Type ty, + mlir::ConversionPatternRewriter &rewriter, + mlir::Value base, ARGS... args) const { + llvm::SmallVector cv = {args...}; + auto llvmPtrTy = + mlir::LLVM::LLVMPointerType::get(ty.getContext(), /*addressSpace=*/0); + return rewriter.create(loc, llvmPtrTy, ty, base, cv); + } + + // Find the Block in which the alloca should be inserted. + // The order to recursively find the proper block: + // 1. An OpenMP Op that will be outlined. + // 2. A LLVMFuncOp + // 3. The first ancestor that is an OpenMP Op or a LLVMFuncOp + mlir::Block *getBlockForAllocaInsert(mlir::Operation *op) const; + + // Generate an alloca of size 1 for an object of type \p llvmObjectTy in the + // allocation address space provided for the architecture in the DataLayout + // specification. If the address space is different from the devices + // program address space we perform a cast. In the case of most architectures + // the program and allocation address space will be the default of 0 and no + // cast will be emitted. + mlir::Value + genAllocaAndAddrCastWithType(mlir::Location loc, mlir::Type llvmObjectTy, + unsigned alignment, + mlir::ConversionPatternRewriter &rewriter) const; + + const fir::LLVMTypeConverter &lowerTy() const { + return *static_cast( + this->getTypeConverter()); + } + + void attachTBAATag(mlir::LLVM::AliasAnalysisOpInterface op, + mlir::Type baseFIRType, mlir::Type accessFIRType, + mlir::LLVM::GEPOp gep) const { + lowerTy().attachTBAATag(op, baseFIRType, accessFIRType, gep); + } + + unsigned + getAllocaAddressSpace(mlir::ConversionPatternRewriter &rewriter) const; + + unsigned + getProgramAddressSpace(mlir::ConversionPatternRewriter &rewriter) const; + + const fir::FIRToLLVMPassOptions &options; + + using ConvertToLLVMPattern::match; + using ConvertToLLVMPattern::matchAndRewrite; +}; + +template +class FIROpConversion : public ConvertFIRToLLVMPattern { +public: + using OpAdaptor = typename SourceOp::Adaptor; + + explicit FIROpConversion(const LLVMTypeConverter &typeConverter, + const fir::FIRToLLVMPassOptions &options, + mlir::PatternBenefit benefit = 1) + : ConvertFIRToLLVMPattern(SourceOp::getOperationName(), + &typeConverter.getContext(), typeConverter, + options, benefit) {} + + /// Wrappers around the RewritePattern methods that pass the derived op type. + void rewrite(mlir::Operation *op, mlir::ArrayRef operands, + mlir::ConversionPatternRewriter &rewriter) const final { + rewrite(mlir::cast(op), + OpAdaptor(operands, mlir::cast(op)), rewriter); + } + mlir::LogicalResult match(mlir::Operation *op) const final { + return match(mlir::cast(op)); + } + mlir::LogicalResult + matchAndRewrite(mlir::Operation *op, mlir::ArrayRef operands, + mlir::ConversionPatternRewriter &rewriter) const final { + return matchAndRewrite(mlir::cast(op), + OpAdaptor(operands, mlir::cast(op)), + rewriter); + } + + /// Rewrite and Match methods that operate on the SourceOp type. These must be + /// overridden by the derived pattern class. + virtual mlir::LogicalResult match(SourceOp op) const { + llvm_unreachable("must override match or matchAndRewrite"); + } + virtual void rewrite(SourceOp op, OpAdaptor adaptor, + mlir::ConversionPatternRewriter &rewriter) const { + llvm_unreachable("must override rewrite or matchAndRewrite"); + } + virtual mlir::LogicalResult + matchAndRewrite(SourceOp op, OpAdaptor adaptor, + mlir::ConversionPatternRewriter &rewriter) const { + if (mlir::failed(match(op))) + return mlir::failure(); + rewrite(op, adaptor, rewriter); + return mlir::success(); + } + +private: + using ConvertFIRToLLVMPattern::matchAndRewrite; + using ConvertToLLVMPattern::match; +}; + +/// FIR conversion pattern template +template +class FIROpAndTypeConversion : public FIROpConversion { +public: + using FIROpConversion::FIROpConversion; + using OpAdaptor = typename FromOp::Adaptor; + + mlir::LogicalResult + matchAndRewrite(FromOp op, OpAdaptor adaptor, + mlir::ConversionPatternRewriter &rewriter) const final { + mlir::Type ty = this->convertType(op.getType()); + return doRewrite(op, ty, adaptor, rewriter); + } + + virtual mlir::LogicalResult + doRewrite(FromOp addr, mlir::Type ty, OpAdaptor adaptor, + mlir::ConversionPatternRewriter &rewriter) const = 0; +}; + +} // namespace fir + +#endif // FORTRAN_OPTIMIZER_CODEGEN_FIROPPATTERNS_H diff --git a/flang/include/flang/Optimizer/CodeGen/TypeConverter.h b/flang/include/flang/Optimizer/CodeGen/TypeConverter.h index 396c13639255..79b3bfe4e80e 100644 --- a/flang/include/flang/Optimizer/CodeGen/TypeConverter.h +++ b/flang/include/flang/Optimizer/CodeGen/TypeConverter.h @@ -94,8 +94,8 @@ public: // to LLVM IR dialect here. // // fir.complex | std.complex --> llvm<"{t,t}"> - template mlir::Type convertComplexType(C cmplx) const { - LLVM_DEBUG(llvm::dbgs() << "type convert: " << cmplx << '\n'); + template + mlir::Type convertComplexType(C cmplx) const { auto eleTy = cmplx.getElementType(); return convertType(specifics->complexMemoryType(eleTy)); } diff --git a/flang/lib/Optimizer/CodeGen/CMakeLists.txt b/flang/lib/Optimizer/CodeGen/CMakeLists.txt index 175ab9fefda2..879bc28d017a 100644 --- a/flang/lib/Optimizer/CodeGen/CMakeLists.txt +++ b/flang/lib/Optimizer/CodeGen/CMakeLists.txt @@ -3,6 +3,7 @@ add_flang_library(FIRCodeGen CGOps.cpp CodeGen.cpp CodeGenOpenMP.cpp + FIROpPatterns.cpp PreCGRewrite.cpp TBAABuilder.cpp Target.cpp diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp index faf90ef6b50a..06ce84f1543a 100644 --- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp +++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp @@ -14,6 +14,8 @@ #include "CGOps.h" #include "flang/Optimizer/CodeGen/CodeGenOpenMP.h" +#include "flang/Optimizer/CodeGen/FIROpPatterns.h" +#include "flang/Optimizer/CodeGen/TypeConverter.h" #include "flang/Optimizer/Dialect/FIRAttr.h" #include "flang/Optimizer/Dialect/FIROps.h" #include "flang/Optimizer/Dialect/FIRType.h" @@ -58,42 +60,14 @@ namespace fir { #define DEBUG_TYPE "flang-codegen" -// fir::LLVMTypeConverter for converting to LLVM IR dialect types. -#include "flang/Optimizer/CodeGen/TypeConverter.h" - // TODO: This should really be recovered from the specified target. static constexpr unsigned defaultAlign = 8; -static constexpr unsigned defaultAddressSpace = 0u; /// `fir.box` attribute values as defined for CFI_attribute_t in /// flang/ISO_Fortran_binding.h. static constexpr unsigned kAttrPointer = CFI_attribute_pointer; static constexpr unsigned kAttrAllocatable = CFI_attribute_allocatable; -static inline unsigned -getAllocaAddressSpace(mlir::ConversionPatternRewriter &rewriter) { - mlir::Operation *parentOp = rewriter.getInsertionBlock()->getParentOp(); - assert(parentOp != nullptr && - "expected insertion block to have parent operation"); - if (auto module = parentOp->getParentOfType()) - if (mlir::Attribute addrSpace = - mlir::DataLayout(module).getAllocaMemorySpace()) - return llvm::cast(addrSpace).getUInt(); - return defaultAddressSpace; -} - -static inline unsigned -getProgramAddressSpace(mlir::ConversionPatternRewriter &rewriter) { - mlir::Operation *parentOp = rewriter.getInsertionBlock()->getParentOp(); - assert(parentOp != nullptr && - "expected insertion block to have parent operation"); - if (auto module = parentOp->getParentOfType()) - if (mlir::Attribute addrSpace = - mlir::DataLayout(module).getProgramMemorySpace()) - return llvm::cast(addrSpace).getUInt(); - return defaultAddressSpace; -} - static inline mlir::Type getLlvmPtrType(mlir::MLIRContext *context, unsigned addressSpace = 0) { return mlir::LLVM::LLVMPointerType::get(context, addressSpace); @@ -151,333 +125,9 @@ static unsigned getLenParamFieldId(mlir::Type ty) { return getTypeDescFieldId(ty) + 1; } -namespace { -/// FIR conversion pattern template -template -class FIROpConversion : public mlir::ConvertOpToLLVMPattern { -public: - explicit FIROpConversion(const fir::LLVMTypeConverter &lowering, - const fir::FIRToLLVMPassOptions &options) - : mlir::ConvertOpToLLVMPattern(lowering), options(options) {} - -protected: - mlir::Type convertType(mlir::Type ty) const { - return lowerTy().convertType(ty); - } - - // Convert FIR type to LLVM without turning fir.box into memory - // reference. - mlir::Type convertObjectType(mlir::Type firType) const { - if (auto boxTy = firType.dyn_cast()) - return lowerTy().convertBoxTypeAsStruct(boxTy); - return lowerTy().convertType(firType); - } - - mlir::LLVM::ConstantOp - genI32Constant(mlir::Location loc, mlir::ConversionPatternRewriter &rewriter, - int value) const { - mlir::Type i32Ty = rewriter.getI32Type(); - mlir::IntegerAttr attr = rewriter.getI32IntegerAttr(value); - return rewriter.create(loc, i32Ty, attr); - } - - mlir::LLVM::ConstantOp - genConstantOffset(mlir::Location loc, - mlir::ConversionPatternRewriter &rewriter, - int offset) const { - mlir::Type ity = lowerTy().offsetType(); - mlir::IntegerAttr cattr = rewriter.getI32IntegerAttr(offset); - return rewriter.create(loc, ity, cattr); - } - - /// Perform an extension or truncation as needed on an integer value. Lowering - /// to the specific target may involve some sign-extending or truncation of - /// values, particularly to fit them from abstract box types to the - /// appropriate reified structures. - mlir::Value integerCast(mlir::Location loc, - mlir::ConversionPatternRewriter &rewriter, - mlir::Type ty, mlir::Value val) const { - auto valTy = val.getType(); - // If the value was not yet lowered, lower its type so that it can - // be used in getPrimitiveTypeSizeInBits. - if (!valTy.isa()) - valTy = convertType(valTy); - auto toSize = mlir::LLVM::getPrimitiveTypeSizeInBits(ty); - auto fromSize = mlir::LLVM::getPrimitiveTypeSizeInBits(valTy); - if (toSize < fromSize) - return rewriter.create(loc, ty, val); - if (toSize > fromSize) - return rewriter.create(loc, ty, val); - return val; - } - - struct TypePair { - mlir::Type fir; - mlir::Type llvm; - }; - - TypePair getBoxTypePair(mlir::Type firBoxTy) const { - mlir::Type llvmBoxTy = lowerTy().convertBoxTypeAsStruct( - mlir::cast(firBoxTy)); - return TypePair{firBoxTy, llvmBoxTy}; - } - - /// Construct code sequence to extract the specific value from a `fir.box`. - mlir::Value getValueFromBox(mlir::Location loc, TypePair boxTy, - mlir::Value box, mlir::Type resultTy, - mlir::ConversionPatternRewriter &rewriter, - int boxValue) const { - if (box.getType().isa()) { - auto pty = ::getLlvmPtrType(resultTy.getContext()); - auto p = rewriter.create( - loc, pty, boxTy.llvm, box, - llvm::ArrayRef{0, boxValue}); - auto loadOp = rewriter.create(loc, resultTy, p); - attachTBAATag(loadOp, boxTy.fir, nullptr, p); - return loadOp; - } - return rewriter.create(loc, box, boxValue); - } - - /// Method to construct code sequence to get the triple for dimension `dim` - /// from a box. - llvm::SmallVector - getDimsFromBox(mlir::Location loc, llvm::ArrayRef retTys, - TypePair boxTy, mlir::Value box, mlir::Value dim, - mlir::ConversionPatternRewriter &rewriter) const { - mlir::Value l0 = - loadDimFieldFromBox(loc, boxTy, box, dim, 0, retTys[0], rewriter); - mlir::Value l1 = - loadDimFieldFromBox(loc, boxTy, box, dim, 1, retTys[1], rewriter); - mlir::Value l2 = - loadDimFieldFromBox(loc, boxTy, box, dim, 2, retTys[2], rewriter); - return {l0, l1, l2}; - } - - llvm::SmallVector - getDimsFromBox(mlir::Location loc, llvm::ArrayRef retTys, - TypePair boxTy, mlir::Value box, int dim, - mlir::ConversionPatternRewriter &rewriter) const { - mlir::Value l0 = - getDimFieldFromBox(loc, boxTy, box, dim, 0, retTys[0], rewriter); - mlir::Value l1 = - getDimFieldFromBox(loc, boxTy, box, dim, 1, retTys[1], rewriter); - mlir::Value l2 = - getDimFieldFromBox(loc, boxTy, box, dim, 2, retTys[2], rewriter); - return {l0, l1, l2}; - } - - mlir::Value - loadDimFieldFromBox(mlir::Location loc, TypePair boxTy, mlir::Value box, - mlir::Value dim, int off, mlir::Type ty, - mlir::ConversionPatternRewriter &rewriter) const { - assert(box.getType().isa() && - "descriptor inquiry with runtime dim can only be done on descriptor " - "in memory"); - mlir::LLVM::GEPOp p = genGEP(loc, boxTy.llvm, rewriter, box, 0, - static_cast(kDimsPosInBox), dim, off); - auto loadOp = rewriter.create(loc, ty, p); - attachTBAATag(loadOp, boxTy.fir, nullptr, p); - return loadOp; - } - - mlir::Value - getDimFieldFromBox(mlir::Location loc, TypePair boxTy, mlir::Value box, - int dim, int off, mlir::Type ty, - mlir::ConversionPatternRewriter &rewriter) const { - if (box.getType().isa()) { - mlir::LLVM::GEPOp p = genGEP(loc, boxTy.llvm, rewriter, box, 0, - static_cast(kDimsPosInBox), dim, off); - auto loadOp = rewriter.create(loc, ty, p); - attachTBAATag(loadOp, boxTy.fir, nullptr, p); - return loadOp; - } - return rewriter.create( - loc, box, llvm::ArrayRef{kDimsPosInBox, dim, off}); - } - - mlir::Value - getStrideFromBox(mlir::Location loc, TypePair boxTy, mlir::Value box, - unsigned dim, - mlir::ConversionPatternRewriter &rewriter) const { - auto idxTy = lowerTy().indexType(); - return getDimFieldFromBox(loc, boxTy, box, dim, kDimStridePos, idxTy, - rewriter); - } - - /// Read base address from a fir.box. Returned address has type ty. - mlir::Value - getBaseAddrFromBox(mlir::Location loc, TypePair boxTy, mlir::Value box, - mlir::ConversionPatternRewriter &rewriter) const { - mlir::Type resultTy = ::getLlvmPtrType(boxTy.llvm.getContext()); - return getValueFromBox(loc, boxTy, box, resultTy, rewriter, kAddrPosInBox); - } - - mlir::Value - getElementSizeFromBox(mlir::Location loc, mlir::Type resultTy, TypePair boxTy, - mlir::Value box, - mlir::ConversionPatternRewriter &rewriter) const { - return getValueFromBox(loc, boxTy, box, resultTy, rewriter, - kElemLenPosInBox); - } - - // Get the element type given an LLVM type that is of the form - // (array|struct|vector)+ and the provided indexes. - static mlir::Type getBoxEleTy(mlir::Type type, - llvm::ArrayRef indexes) { - for (unsigned i : indexes) { - if (auto t = type.dyn_cast()) { - assert(!t.isOpaque() && i < t.getBody().size()); - type = t.getBody()[i]; - } else if (auto t = type.dyn_cast()) { - type = t.getElementType(); - } else if (auto t = type.dyn_cast()) { - type = t.getElementType(); - } else { - fir::emitFatalError(mlir::UnknownLoc::get(type.getContext()), - "request for invalid box element type"); - } - } - return type; - } - - // Return LLVM type of the object described by a fir.box of \p boxType. - mlir::Type getLlvmObjectTypeFromBoxType(mlir::Type boxType) const { - mlir::Type objectType = fir::dyn_cast_ptrOrBoxEleTy(boxType); - assert(objectType && "boxType must be a box type"); - return this->convertType(objectType); - } - - /// Read the address of the type descriptor from a box. - mlir::Value - loadTypeDescAddress(mlir::Location loc, TypePair boxTy, mlir::Value box, - mlir::ConversionPatternRewriter &rewriter) const { - unsigned typeDescFieldId = getTypeDescFieldId(boxTy.fir); - mlir::Type tdescType = lowerTy().convertTypeDescType(rewriter.getContext()); - return getValueFromBox(loc, boxTy, box, tdescType, rewriter, - typeDescFieldId); - } - - // Load the attribute from the \p box and perform a check against \p maskValue - // The final comparison is implemented as `(attribute & maskValue) != 0`. - mlir::Value genBoxAttributeCheck(mlir::Location loc, TypePair boxTy, - mlir::Value box, - mlir::ConversionPatternRewriter &rewriter, - unsigned maskValue) const { - mlir::Type attrTy = rewriter.getI32Type(); - mlir::Value attribute = - getValueFromBox(loc, boxTy, box, attrTy, rewriter, kAttributePosInBox); - mlir::LLVM::ConstantOp attrMask = - genConstantOffset(loc, rewriter, maskValue); - auto maskRes = - rewriter.create(loc, attrTy, attribute, attrMask); - mlir::LLVM::ConstantOp c0 = genConstantOffset(loc, rewriter, 0); - return rewriter.create( - loc, mlir::LLVM::ICmpPredicate::ne, maskRes, c0); - } - - template - mlir::LLVM::GEPOp genGEP(mlir::Location loc, mlir::Type ty, - mlir::ConversionPatternRewriter &rewriter, - mlir::Value base, ARGS... args) const { - llvm::SmallVector cv = {args...}; - auto llvmPtrTy = ::getLlvmPtrType(ty.getContext()); - return rewriter.create(loc, llvmPtrTy, ty, base, cv); - } - - // Find the Block in which the alloca should be inserted. - // The order to recursively find the proper block: - // 1. An OpenMP Op that will be outlined. - // 2. A LLVMFuncOp - // 3. The first ancestor that is an OpenMP Op or a LLVMFuncOp - static mlir::Block *getBlockForAllocaInsert(mlir::Operation *op) { - if (auto iface = - mlir::dyn_cast(op)) - return iface.getAllocaBlock(); - if (auto llvmFuncOp = mlir::dyn_cast(op)) - return &llvmFuncOp.front(); - return getBlockForAllocaInsert(op->getParentOp()); - } - - // Generate an alloca of size 1 for an object of type \p llvmObjectTy in the - // allocation address space provided for the architecture in the DataLayout - // specification. If the address space is different from the devices - // program address space we perform a cast. In the case of most architectures - // the program and allocation address space will be the default of 0 and no - // cast will be emitted. - mlir::Value genAllocaAndAddrCastWithType( - mlir::Location loc, mlir::Type llvmObjectTy, unsigned alignment, - mlir::ConversionPatternRewriter &rewriter) const { - auto thisPt = rewriter.saveInsertionPoint(); - mlir::Operation *parentOp = rewriter.getInsertionBlock()->getParentOp(); - if (mlir::isa(parentOp)) { - // DeclareReductionOp has multiple child regions. We want to get the first - // block of whichever of those regions we are currently in - mlir::Region *parentRegion = rewriter.getInsertionBlock()->getParent(); - rewriter.setInsertionPointToStart(&parentRegion->front()); - } else { - mlir::Block *insertBlock = getBlockForAllocaInsert(parentOp); - rewriter.setInsertionPointToStart(insertBlock); - } - auto size = genI32Constant(loc, rewriter, 1); - unsigned allocaAs = getAllocaAddressSpace(rewriter); - unsigned programAs = getProgramAddressSpace(rewriter); - - mlir::Value al = rewriter.create( - loc, ::getLlvmPtrType(llvmObjectTy.getContext(), allocaAs), - llvmObjectTy, size, alignment); - - // if our allocation address space, is not the same as the program address - // space, then we must emit a cast to the program address space before use. - // An example case would be on AMDGPU, where the allocation address space is - // the numeric value 5 (private), and the program address space is 0 - // (generic). - if (allocaAs != programAs) { - al = rewriter.create( - loc, ::getLlvmPtrType(llvmObjectTy.getContext(), programAs), al); - } - - rewriter.restoreInsertionPoint(thisPt); - return al; - } - - const fir::LLVMTypeConverter &lowerTy() const { - return *static_cast( - this->getTypeConverter()); - } - - void attachTBAATag(mlir::LLVM::AliasAnalysisOpInterface op, - mlir::Type baseFIRType, mlir::Type accessFIRType, - mlir::LLVM::GEPOp gep) const { - lowerTy().attachTBAATag(op, baseFIRType, accessFIRType, gep); - } - - const fir::FIRToLLVMPassOptions &options; -}; - -/// FIR conversion pattern template -template -class FIROpAndTypeConversion : public FIROpConversion { -public: - using FIROpConversion::FIROpConversion; - using OpAdaptor = typename FromOp::Adaptor; - - mlir::LogicalResult - matchAndRewrite(FromOp op, OpAdaptor adaptor, - mlir::ConversionPatternRewriter &rewriter) const final { - mlir::Type ty = this->convertType(op.getType()); - return doRewrite(op, ty, adaptor, rewriter); - } - - virtual mlir::LogicalResult - doRewrite(FromOp addr, mlir::Type ty, OpAdaptor adaptor, - mlir::ConversionPatternRewriter &rewriter) const = 0; -}; -} // namespace - namespace { /// Lower `fir.address_of` operation to `llvm.address_of` operation. -struct AddrOfOpConversion : public FIROpConversion { +struct AddrOfOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -539,7 +189,7 @@ genAllocationScaleSize(OP op, mlir::Type ity, namespace { /// convert to LLVM IR dialect `alloca` -struct AllocaOpConversion : public FIROpConversion { +struct AllocaOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -624,7 +274,7 @@ struct AllocaOpConversion : public FIROpConversion { namespace { /// Lower `fir.box_addr` to the sequence of operations to extract the first /// element of the box. -struct BoxAddrOpConversion : public FIROpConversion { +struct BoxAddrOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -645,7 +295,7 @@ struct BoxAddrOpConversion : public FIROpConversion { /// Convert `!fir.boxchar_len` to `!llvm.extractvalue` for the 2nd part of the /// boxchar. -struct BoxCharLenOpConversion : public FIROpConversion { +struct BoxCharLenOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -668,7 +318,7 @@ struct BoxCharLenOpConversion : public FIROpConversion { /// Lower `fir.box_dims` to a sequence of operations to extract the requested /// dimension information from the boxed value. /// Result in a triple set of GEPs and loads. -struct BoxDimsOpConversion : public FIROpConversion { +struct BoxDimsOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -690,7 +340,7 @@ struct BoxDimsOpConversion : public FIROpConversion { /// Lower `fir.box_elesize` to a sequence of operations ro extract the size of /// an element in the boxed value. -struct BoxEleSizeOpConversion : public FIROpConversion { +struct BoxEleSizeOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -708,7 +358,7 @@ struct BoxEleSizeOpConversion : public FIROpConversion { /// Lower `fir.box_isalloc` to a sequence of operations to determine if the /// boxed value was from an ALLOCATABLE entity. -struct BoxIsAllocOpConversion : public FIROpConversion { +struct BoxIsAllocOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -726,7 +376,7 @@ struct BoxIsAllocOpConversion : public FIROpConversion { /// Lower `fir.box_isarray` to a sequence of operations to determine if the /// boxed is an array. -struct BoxIsArrayOpConversion : public FIROpConversion { +struct BoxIsArrayOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -746,7 +396,7 @@ struct BoxIsArrayOpConversion : public FIROpConversion { /// Lower `fir.box_isptr` to a sequence of operations to determined if the /// boxed value was from a POINTER entity. -struct BoxIsPtrOpConversion : public FIROpConversion { +struct BoxIsPtrOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -764,7 +414,7 @@ struct BoxIsPtrOpConversion : public FIROpConversion { /// Lower `fir.box_rank` to the sequence of operation to extract the rank from /// the box. -struct BoxRankOpConversion : public FIROpConversion { +struct BoxRankOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -784,7 +434,8 @@ struct BoxRankOpConversion : public FIROpConversion { /// Lower `fir.boxproc_host` operation. Extracts the host pointer from the /// boxproc. /// TODO: Part of supporting Fortran 2003 procedure pointers. -struct BoxProcHostOpConversion : public FIROpConversion { +struct BoxProcHostOpConversion + : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -797,7 +448,8 @@ struct BoxProcHostOpConversion : public FIROpConversion { /// Lower `fir.box_tdesc` to the sequence of operations to extract the type /// descriptor from the box. -struct BoxTypeDescOpConversion : public FIROpConversion { +struct BoxTypeDescOpConversion + : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -814,7 +466,8 @@ struct BoxTypeDescOpConversion : public FIROpConversion { /// Lower `fir.box_typecode` to a sequence of operations to extract the type /// code in the boxed value. -struct BoxTypeCodeOpConversion : public FIROpConversion { +struct BoxTypeCodeOpConversion + : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -832,7 +485,7 @@ struct BoxTypeCodeOpConversion : public FIROpConversion { }; /// Lower `fir.string_lit` to LLVM IR dialect operation. -struct StringLitOpConversion : public FIROpConversion { +struct StringLitOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -872,7 +525,7 @@ struct StringLitOpConversion : public FIROpConversion { }; /// `fir.call` -> `llvm.call` -struct CallOpConversion : public FIROpConversion { +struct CallOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -903,7 +556,7 @@ namespace { /// Per 10.1, the only comparisons available are .EQ. (oeq) and .NE. (une). /// /// For completeness, all other comparison are done on the real component only. -struct CmpcOpConversion : public FIROpConversion { +struct CmpcOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -941,7 +594,7 @@ struct CmpcOpConversion : public FIROpConversion { }; /// Lower complex constants -struct ConstcOpConversion : public FIROpConversion { +struct ConstcOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -968,7 +621,7 @@ struct ConstcOpConversion : public FIROpConversion { }; /// convert value of from-type to value of to-type -struct ConvertOpConversion : public FIROpConversion { +struct ConvertOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; static bool isFloatingPointTy(mlir::Type ty) { @@ -1137,7 +790,7 @@ struct ConvertOpConversion : public FIROpConversion { /// only used to carry information during FIR to FIR passes. It may be used /// in the future to generate the runtime type info data structures instead /// of generating them in lowering. -struct TypeInfoOpConversion : public FIROpConversion { +struct TypeInfoOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -1150,7 +803,7 @@ struct TypeInfoOpConversion : public FIROpConversion { /// `fir.dt_entry` operation has no specific CodeGen. The operation is only used /// to carry information during FIR to FIR passes. -struct DTEntryOpConversion : public FIROpConversion { +struct DTEntryOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -1162,7 +815,7 @@ struct DTEntryOpConversion : public FIROpConversion { }; /// Lower `fir.global_len` operation. -struct GlobalLenOpConversion : public FIROpConversion { +struct GlobalLenOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -1175,7 +828,7 @@ struct GlobalLenOpConversion : public FIROpConversion { /// Lower fir.len_param_index struct LenParamIndexOpConversion - : public FIROpConversion { + : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; // FIXME: this should be specialized by the runtime target @@ -1190,7 +843,7 @@ struct LenParamIndexOpConversion /// instructions that generate `!llvm.struct<(ptr, i64)>`. The 1st element /// in this struct is a pointer. Its type is determined from `KIND`. The 2nd /// element is the length of the character buffer (`#n`). -struct EmboxCharOpConversion : public FIROpConversion { +struct EmboxCharOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -1283,7 +936,7 @@ genTypeStrideInBytes(mlir::Location loc, mlir::Type idxTy, namespace { /// Lower a `fir.allocmem` instruction into `llvm.call @malloc` -struct AllocMemOpConversion : public FIROpConversion { +struct AllocMemOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -1352,7 +1005,7 @@ static unsigned getDimension(mlir::LLVM::LLVMArrayType ty) { namespace { /// Lower a `fir.freemem` instruction into `llvm.call @free` -struct FreeMemOpConversion : public FIROpConversion { +struct FreeMemOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -1410,9 +1063,9 @@ convertSubcomponentIndices(mlir::Location loc, mlir::Type eleTy, /// Common base class for embox to descriptor conversion. template -struct EmboxCommonConversion : public FIROpConversion { - using FIROpConversion::FIROpConversion; - using TypePair = typename FIROpConversion::TypePair; +struct EmboxCommonConversion : public fir::FIROpConversion { + using fir::FIROpConversion::FIROpConversion; + using TypePair = typename fir::FIROpConversion::TypePair; static int getCFIAttr(fir::BaseBoxType boxTy) { auto eleTy = boxTy.getEleTy(); @@ -1434,8 +1087,8 @@ struct EmboxCommonConversion : public FIROpConversion { return size; // Length accounted for in the genTypeStrideInBytes GEP. // Otherwise, multiply the single character size by the length. assert(!lenParams.empty()); - auto len64 = FIROpConversion::integerCast(loc, rewriter, i64Ty, - lenParams.back()); + auto len64 = fir::FIROpConversion::integerCast(loc, rewriter, i64Ty, + lenParams.back()); return rewriter.create(loc, i64Ty, size, len64); } @@ -2280,7 +1933,7 @@ private: /// Lower `fir.emboxproc` operation. Creates a procedure box. /// TODO: Part of supporting Fortran 2003 procedure pointers. -struct EmboxProcOpConversion : public FIROpConversion { +struct EmboxProcOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -2346,7 +1999,7 @@ private: namespace { /// Extract a subobject value from an ssa-value of aggregate type struct ExtractValueOpConversion - : public FIROpAndTypeConversion, + : public fir::FIROpAndTypeConversion, public ValueOpCommon { using FIROpAndTypeConversion::FIROpAndTypeConversion; @@ -2365,7 +2018,7 @@ struct ExtractValueOpConversion /// InsertValue is the generalized instruction for the composition of new /// aggregate type values. struct InsertValueOpConversion - : public FIROpAndTypeConversion, + : public fir::FIROpAndTypeConversion, public ValueOpCommon { using FIROpAndTypeConversion::FIROpAndTypeConversion; @@ -2383,7 +2036,7 @@ struct InsertValueOpConversion /// InsertOnRange inserts a value into a sequence over a range of offsets. struct InsertOnRangeOpConversion - : public FIROpAndTypeConversion { + : public fir::FIROpAndTypeConversion { using FIROpAndTypeConversion::FIROpAndTypeConversion; // Increments an array of subscripts in a row major fasion. @@ -2447,7 +2100,7 @@ namespace { /// (See the static restriction on coordinate_of.) array_coor determines the /// coordinate (location) of a specific element. struct XArrayCoorOpConversion - : public FIROpAndTypeConversion { + : public fir::FIROpAndTypeConversion { using FIROpAndTypeConversion::FIROpAndTypeConversion; mlir::LogicalResult @@ -2615,7 +2268,7 @@ struct XArrayCoorOpConversion /// With unboxed arrays, there is the restriction that the array have a static /// shape in all but the last column. struct CoordinateOpConversion - : public FIROpAndTypeConversion { + : public fir::FIROpAndTypeConversion { using FIROpAndTypeConversion::FIROpAndTypeConversion; mlir::LogicalResult @@ -2910,7 +2563,7 @@ private: /// Convert `fir.field_index`. The conversion depends on whether the size of /// the record is static or dynamic. -struct FieldIndexOpConversion : public FIROpConversion { +struct FieldIndexOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; // NB: most field references should be resolved by this point @@ -2951,7 +2604,7 @@ struct FieldIndexOpConversion : public FIROpConversion { }; /// Convert `fir.end` -struct FirEndOpConversion : public FIROpConversion { +struct FirEndOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -2963,7 +2616,7 @@ struct FirEndOpConversion : public FIROpConversion { }; /// Lower `fir.type_desc` to a global addr. -struct TypeDescOpConversion : public FIROpConversion { +struct TypeDescOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -2990,7 +2643,7 @@ struct TypeDescOpConversion : public FIROpConversion { }; /// Lower `fir.has_value` operation to `llvm.return` operation. -struct HasValueOpConversion : public FIROpConversion { +struct HasValueOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3043,7 +2696,7 @@ static inline bool attributeTypeIsCompatible(mlir::MLIRContext *ctx, /// Lower `fir.global` operation to `llvm.global` operation. /// `fir.insert_on_range` operations are replaced with constant dense attribute /// if they are applied on the full range. -struct GlobalOpConversion : public FIROpConversion { +struct GlobalOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3167,7 +2820,7 @@ private: }; /// `fir.load` --> `llvm.load` -struct LoadOpConversion : public FIROpConversion { +struct LoadOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3216,7 +2869,7 @@ struct LoadOpConversion : public FIROpConversion { /// Lower `fir.no_reassoc` to LLVM IR dialect. /// TODO: how do we want to enforce this in LLVM-IR? Can we manipulate the fast /// math flags? -struct NoReassocOpConversion : public FIROpConversion { +struct NoReassocOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3277,7 +2930,7 @@ static void genCaseLadderStep(mlir::Location loc, mlir::Value cmp, /// upper bound in the same case condition. /// /// TODO: lowering of CHARACTER type cases is not handled yet. -struct SelectCaseOpConversion : public FIROpConversion { +struct SelectCaseOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3391,7 +3044,7 @@ static void selectMatchAndRewrite(const fir::LLVMTypeConverter &lowering, } /// conversion of fir::SelectOp to an if-then-else ladder -struct SelectOpConversion : public FIROpConversion { +struct SelectOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3403,7 +3056,7 @@ struct SelectOpConversion : public FIROpConversion { }; /// conversion of fir::SelectRankOp to an if-then-else ladder -struct SelectRankOpConversion : public FIROpConversion { +struct SelectRankOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3415,7 +3068,7 @@ struct SelectRankOpConversion : public FIROpConversion { }; /// Lower `fir.select_type` to LLVM IR dialect. -struct SelectTypeOpConversion : public FIROpConversion { +struct SelectTypeOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3428,7 +3081,7 @@ struct SelectTypeOpConversion : public FIROpConversion { }; /// `fir.store` --> `llvm.store` -struct StoreOpConversion : public FIROpConversion { +struct StoreOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3462,7 +3115,7 @@ namespace { /// Convert `fir.unboxchar` into two `llvm.extractvalue` instructions. One for /// the character buffer and one for the buffer length. -struct UnboxCharOpConversion : public FIROpConversion { +struct UnboxCharOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3487,7 +3140,7 @@ struct UnboxCharOpConversion : public FIROpConversion { /// Lower `fir.unboxproc` operation. Unbox a procedure box value, yielding its /// components. /// TODO: Part of supporting Fortran 2003 procedure pointers. -struct UnboxProcOpConversion : public FIROpConversion { +struct UnboxProcOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3499,7 +3152,7 @@ struct UnboxProcOpConversion : public FIROpConversion { }; /// convert to LLVM IR dialect `undef` -struct UndefOpConversion : public FIROpConversion { +struct UndefOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3511,7 +3164,7 @@ struct UndefOpConversion : public FIROpConversion { } }; -struct ZeroOpConversion : public FIROpConversion { +struct ZeroOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3524,7 +3177,8 @@ struct ZeroOpConversion : public FIROpConversion { }; /// `fir.unreachable` --> `llvm.unreachable` -struct UnreachableOpConversion : public FIROpConversion { +struct UnreachableOpConversion + : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3541,7 +3195,7 @@ struct UnreachableOpConversion : public FIROpConversion { /// %1 = llvm.ptrtoint %0 /// %2 = llvm.icmp "ne" %1, %0 : i64 /// ``` -struct IsPresentOpConversion : public FIROpConversion { +struct IsPresentOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3570,7 +3224,7 @@ struct IsPresentOpConversion : public FIROpConversion { /// Create value signaling an absent optional argument in a call, e.g. /// `fir.absent !fir.ref` --> `llvm.mlir.zero : !llvm.ptr` -struct AbsentOpConversion : public FIROpConversion { +struct AbsentOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3630,7 +3284,7 @@ complexSum(OPTY sumop, mlir::ValueRange opnds, } // namespace namespace { -struct AddcOpConversion : public FIROpConversion { +struct AddcOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3645,7 +3299,7 @@ struct AddcOpConversion : public FIROpConversion { } }; -struct SubcOpConversion : public FIROpConversion { +struct SubcOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3661,7 +3315,7 @@ struct SubcOpConversion : public FIROpConversion { }; /// Inlined complex multiply -struct MulcOpConversion : public FIROpConversion { +struct MulcOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3695,7 +3349,7 @@ struct MulcOpConversion : public FIROpConversion { }; /// Inlined complex division -struct DivcOpConversion : public FIROpConversion { +struct DivcOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3735,7 +3389,7 @@ struct DivcOpConversion : public FIROpConversion { }; /// Inlined complex negation -struct NegcOpConversion : public FIROpConversion { +struct NegcOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3756,7 +3410,7 @@ struct NegcOpConversion : public FIROpConversion { } }; -struct BoxOffsetOpConversion : public FIROpConversion { +struct BoxOffsetOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3782,10 +3436,10 @@ struct BoxOffsetOpConversion : public FIROpConversion { /// anymore uses. /// These operations are normally dead after the pre-codegen pass. template -struct MustBeDeadConversion : public FIROpConversion { +struct MustBeDeadConversion : public fir::FIROpConversion { explicit MustBeDeadConversion(const fir::LLVMTypeConverter &lowering, const fir::FIRToLLVMPassOptions &options) - : FIROpConversion(lowering, options) {} + : fir::FIROpConversion(lowering, options) {} using OpAdaptor = typename FromOp::Adaptor; mlir::LogicalResult @@ -3799,7 +3453,7 @@ struct MustBeDeadConversion : public FIROpConversion { }; struct UnrealizedConversionCastOpConversion - : public FIROpConversion { + : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult diff --git a/flang/lib/Optimizer/CodeGen/FIROpPatterns.cpp b/flang/lib/Optimizer/CodeGen/FIROpPatterns.cpp new file mode 100644 index 000000000000..26871d888815 --- /dev/null +++ b/flang/lib/Optimizer/CodeGen/FIROpPatterns.cpp @@ -0,0 +1,315 @@ +//===-- CodeGen.cpp -- bridge to lower to LLVM ----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Coding style: https://mlir.llvm.org/getting_started/DeveloperGuide/ +// +//===----------------------------------------------------------------------===// + +#include "flang/Optimizer/CodeGen/FIROpPatterns.h" +#include "mlir/Dialect/OpenMP/OpenMPDialect.h" +#include "llvm/Support/Debug.h" + +static inline mlir::Type getLlvmPtrType(mlir::MLIRContext *context, + unsigned addressSpace = 0) { + return mlir::LLVM::LLVMPointerType::get(context, addressSpace); +} + +static unsigned getTypeDescFieldId(mlir::Type ty) { + auto isArray = fir::dyn_cast_ptrOrBoxEleTy(ty).isa(); + return isArray ? kOptTypePtrPosInBox : kDimsPosInBox; +} + +namespace fir { + +ConvertFIRToLLVMPattern::ConvertFIRToLLVMPattern( + llvm::StringRef rootOpName, mlir::MLIRContext *context, + const fir::LLVMTypeConverter &typeConverter, + const fir::FIRToLLVMPassOptions &options, mlir::PatternBenefit benefit) + : ConvertToLLVMPattern(rootOpName, context, typeConverter, benefit), + options(options) {} + +// Convert FIR type to LLVM without turning fir.box into memory +// reference. +mlir::Type +ConvertFIRToLLVMPattern::convertObjectType(mlir::Type firType) const { + if (auto boxTy = firType.dyn_cast()) + return lowerTy().convertBoxTypeAsStruct(boxTy); + return lowerTy().convertType(firType); +} + +mlir::LLVM::ConstantOp ConvertFIRToLLVMPattern::genI32Constant( + mlir::Location loc, mlir::ConversionPatternRewriter &rewriter, + int value) const { + mlir::Type i32Ty = rewriter.getI32Type(); + mlir::IntegerAttr attr = rewriter.getI32IntegerAttr(value); + return rewriter.create(loc, i32Ty, attr); +} + +mlir::LLVM::ConstantOp ConvertFIRToLLVMPattern::genConstantOffset( + mlir::Location loc, mlir::ConversionPatternRewriter &rewriter, + int offset) const { + mlir::Type ity = lowerTy().offsetType(); + mlir::IntegerAttr cattr = rewriter.getI32IntegerAttr(offset); + return rewriter.create(loc, ity, cattr); +} + +/// Perform an extension or truncation as needed on an integer value. Lowering +/// to the specific target may involve some sign-extending or truncation of +/// values, particularly to fit them from abstract box types to the +/// appropriate reified structures. +mlir::Value +ConvertFIRToLLVMPattern::integerCast(mlir::Location loc, + mlir::ConversionPatternRewriter &rewriter, + mlir::Type ty, mlir::Value val) const { + auto valTy = val.getType(); + // If the value was not yet lowered, lower its type so that it can + // be used in getPrimitiveTypeSizeInBits. + if (!valTy.isa()) + valTy = convertType(valTy); + auto toSize = mlir::LLVM::getPrimitiveTypeSizeInBits(ty); + auto fromSize = mlir::LLVM::getPrimitiveTypeSizeInBits(valTy); + if (toSize < fromSize) + return rewriter.create(loc, ty, val); + if (toSize > fromSize) + return rewriter.create(loc, ty, val); + return val; +} + +fir::ConvertFIRToLLVMPattern::TypePair +ConvertFIRToLLVMPattern::getBoxTypePair(mlir::Type firBoxTy) const { + mlir::Type llvmBoxTy = + lowerTy().convertBoxTypeAsStruct(mlir::cast(firBoxTy)); + return TypePair{firBoxTy, llvmBoxTy}; +} + +/// Construct code sequence to extract the specific value from a `fir.box`. +mlir::Value ConvertFIRToLLVMPattern::getValueFromBox( + mlir::Location loc, TypePair boxTy, mlir::Value box, mlir::Type resultTy, + mlir::ConversionPatternRewriter &rewriter, int boxValue) const { + if (box.getType().isa()) { + auto pty = getLlvmPtrType(resultTy.getContext()); + auto p = rewriter.create( + loc, pty, boxTy.llvm, box, + llvm::ArrayRef{0, boxValue}); + auto loadOp = rewriter.create(loc, resultTy, p); + attachTBAATag(loadOp, boxTy.fir, nullptr, p); + return loadOp; + } + return rewriter.create(loc, box, boxValue); +} + +/// Method to construct code sequence to get the triple for dimension `dim` +/// from a box. +llvm::SmallVector ConvertFIRToLLVMPattern::getDimsFromBox( + mlir::Location loc, llvm::ArrayRef retTys, TypePair boxTy, + mlir::Value box, mlir::Value dim, + mlir::ConversionPatternRewriter &rewriter) const { + mlir::Value l0 = + loadDimFieldFromBox(loc, boxTy, box, dim, 0, retTys[0], rewriter); + mlir::Value l1 = + loadDimFieldFromBox(loc, boxTy, box, dim, 1, retTys[1], rewriter); + mlir::Value l2 = + loadDimFieldFromBox(loc, boxTy, box, dim, 2, retTys[2], rewriter); + return {l0, l1, l2}; +} + +llvm::SmallVector ConvertFIRToLLVMPattern::getDimsFromBox( + mlir::Location loc, llvm::ArrayRef retTys, TypePair boxTy, + mlir::Value box, int dim, mlir::ConversionPatternRewriter &rewriter) const { + mlir::Value l0 = + getDimFieldFromBox(loc, boxTy, box, dim, 0, retTys[0], rewriter); + mlir::Value l1 = + getDimFieldFromBox(loc, boxTy, box, dim, 1, retTys[1], rewriter); + mlir::Value l2 = + getDimFieldFromBox(loc, boxTy, box, dim, 2, retTys[2], rewriter); + return {l0, l1, l2}; +} + +mlir::Value ConvertFIRToLLVMPattern::loadDimFieldFromBox( + mlir::Location loc, TypePair boxTy, mlir::Value box, mlir::Value dim, + int off, mlir::Type ty, mlir::ConversionPatternRewriter &rewriter) const { + assert(box.getType().isa() && + "descriptor inquiry with runtime dim can only be done on descriptor " + "in memory"); + mlir::LLVM::GEPOp p = genGEP(loc, boxTy.llvm, rewriter, box, 0, + static_cast(kDimsPosInBox), dim, off); + auto loadOp = rewriter.create(loc, ty, p); + attachTBAATag(loadOp, boxTy.fir, nullptr, p); + return loadOp; +} + +mlir::Value ConvertFIRToLLVMPattern::getDimFieldFromBox( + mlir::Location loc, TypePair boxTy, mlir::Value box, int dim, int off, + mlir::Type ty, mlir::ConversionPatternRewriter &rewriter) const { + if (box.getType().isa()) { + mlir::LLVM::GEPOp p = genGEP(loc, boxTy.llvm, rewriter, box, 0, + static_cast(kDimsPosInBox), dim, off); + auto loadOp = rewriter.create(loc, ty, p); + attachTBAATag(loadOp, boxTy.fir, nullptr, p); + return loadOp; + } + return rewriter.create( + loc, box, llvm::ArrayRef{kDimsPosInBox, dim, off}); +} + +mlir::Value ConvertFIRToLLVMPattern::getStrideFromBox( + mlir::Location loc, TypePair boxTy, mlir::Value box, unsigned dim, + mlir::ConversionPatternRewriter &rewriter) const { + auto idxTy = lowerTy().indexType(); + return getDimFieldFromBox(loc, boxTy, box, dim, kDimStridePos, idxTy, + rewriter); +} + +/// Read base address from a fir.box. Returned address has type ty. +mlir::Value ConvertFIRToLLVMPattern::getBaseAddrFromBox( + mlir::Location loc, TypePair boxTy, mlir::Value box, + mlir::ConversionPatternRewriter &rewriter) const { + mlir::Type resultTy = ::getLlvmPtrType(boxTy.llvm.getContext()); + return getValueFromBox(loc, boxTy, box, resultTy, rewriter, kAddrPosInBox); +} + +mlir::Value ConvertFIRToLLVMPattern::getElementSizeFromBox( + mlir::Location loc, mlir::Type resultTy, TypePair boxTy, mlir::Value box, + mlir::ConversionPatternRewriter &rewriter) const { + return getValueFromBox(loc, boxTy, box, resultTy, rewriter, kElemLenPosInBox); +} + +// Get the element type given an LLVM type that is of the form +// (array|struct|vector)+ and the provided indexes. +mlir::Type ConvertFIRToLLVMPattern::getBoxEleTy( + mlir::Type type, llvm::ArrayRef indexes) const { + for (unsigned i : indexes) { + if (auto t = type.dyn_cast()) { + assert(!t.isOpaque() && i < t.getBody().size()); + type = t.getBody()[i]; + } else if (auto t = type.dyn_cast()) { + type = t.getElementType(); + } else if (auto t = type.dyn_cast()) { + type = t.getElementType(); + } else { + fir::emitFatalError(mlir::UnknownLoc::get(type.getContext()), + "request for invalid box element type"); + } + } + return type; +} + +// Return LLVM type of the object described by a fir.box of \p boxType. +mlir::Type ConvertFIRToLLVMPattern::getLlvmObjectTypeFromBoxType( + mlir::Type boxType) const { + mlir::Type objectType = fir::dyn_cast_ptrOrBoxEleTy(boxType); + assert(objectType && "boxType must be a box type"); + return this->convertType(objectType); +} + +/// Read the address of the type descriptor from a box. +mlir::Value ConvertFIRToLLVMPattern::loadTypeDescAddress( + mlir::Location loc, TypePair boxTy, mlir::Value box, + mlir::ConversionPatternRewriter &rewriter) const { + unsigned typeDescFieldId = getTypeDescFieldId(boxTy.fir); + mlir::Type tdescType = lowerTy().convertTypeDescType(rewriter.getContext()); + return getValueFromBox(loc, boxTy, box, tdescType, rewriter, typeDescFieldId); +} + +// Load the attribute from the \p box and perform a check against \p maskValue +// The final comparison is implemented as `(attribute & maskValue) != 0`. +mlir::Value ConvertFIRToLLVMPattern::genBoxAttributeCheck( + mlir::Location loc, TypePair boxTy, mlir::Value box, + mlir::ConversionPatternRewriter &rewriter, unsigned maskValue) const { + mlir::Type attrTy = rewriter.getI32Type(); + mlir::Value attribute = + getValueFromBox(loc, boxTy, box, attrTy, rewriter, kAttributePosInBox); + mlir::LLVM::ConstantOp attrMask = genConstantOffset(loc, rewriter, maskValue); + auto maskRes = + rewriter.create(loc, attrTy, attribute, attrMask); + mlir::LLVM::ConstantOp c0 = genConstantOffset(loc, rewriter, 0); + return rewriter.create(loc, mlir::LLVM::ICmpPredicate::ne, + maskRes, c0); +} + +// Find the Block in which the alloca should be inserted. +// The order to recursively find the proper block: +// 1. An OpenMP Op that will be outlined. +// 2. A LLVMFuncOp +// 3. The first ancestor that is an OpenMP Op or a LLVMFuncOp +mlir::Block * +ConvertFIRToLLVMPattern::getBlockForAllocaInsert(mlir::Operation *op) const { + if (auto iface = mlir::dyn_cast(op)) + return iface.getAllocaBlock(); + if (auto llvmFuncOp = mlir::dyn_cast(op)) + return &llvmFuncOp.front(); + return getBlockForAllocaInsert(op->getParentOp()); +} + +// Generate an alloca of size 1 for an object of type \p llvmObjectTy in the +// allocation address space provided for the architecture in the DataLayout +// specification. If the address space is different from the devices +// program address space we perform a cast. In the case of most architectures +// the program and allocation address space will be the default of 0 and no +// cast will be emitted. +mlir::Value ConvertFIRToLLVMPattern::genAllocaAndAddrCastWithType( + mlir::Location loc, mlir::Type llvmObjectTy, unsigned alignment, + mlir::ConversionPatternRewriter &rewriter) const { + auto thisPt = rewriter.saveInsertionPoint(); + mlir::Operation *parentOp = rewriter.getInsertionBlock()->getParentOp(); + if (mlir::isa(parentOp)) { + // DeclareReductionOp has multiple child regions. We want to get the first + // block of whichever of those regions we are currently in + mlir::Region *parentRegion = rewriter.getInsertionBlock()->getParent(); + rewriter.setInsertionPointToStart(&parentRegion->front()); + } else { + mlir::Block *insertBlock = getBlockForAllocaInsert(parentOp); + rewriter.setInsertionPointToStart(insertBlock); + } + auto size = genI32Constant(loc, rewriter, 1); + unsigned allocaAs = getAllocaAddressSpace(rewriter); + unsigned programAs = getProgramAddressSpace(rewriter); + + mlir::Value al = rewriter.create( + loc, ::getLlvmPtrType(llvmObjectTy.getContext(), allocaAs), llvmObjectTy, + size, alignment); + + // if our allocation address space, is not the same as the program address + // space, then we must emit a cast to the program address space before use. + // An example case would be on AMDGPU, where the allocation address space is + // the numeric value 5 (private), and the program address space is 0 + // (generic). + if (allocaAs != programAs) { + al = rewriter.create( + loc, ::getLlvmPtrType(llvmObjectTy.getContext(), programAs), al); + } + + rewriter.restoreInsertionPoint(thisPt); + return al; +} + +unsigned ConvertFIRToLLVMPattern::getAllocaAddressSpace( + mlir::ConversionPatternRewriter &rewriter) const { + mlir::Operation *parentOp = rewriter.getInsertionBlock()->getParentOp(); + assert(parentOp != nullptr && + "expected insertion block to have parent operation"); + if (auto module = parentOp->getParentOfType()) + if (mlir::Attribute addrSpace = + mlir::DataLayout(module).getAllocaMemorySpace()) + return llvm::cast(addrSpace).getUInt(); + return defaultAddressSpace; +} + +unsigned ConvertFIRToLLVMPattern::getProgramAddressSpace( + mlir::ConversionPatternRewriter &rewriter) const { + mlir::Operation *parentOp = rewriter.getInsertionBlock()->getParentOp(); + assert(parentOp != nullptr && + "expected insertion block to have parent operation"); + if (auto module = parentOp->getParentOfType()) + if (mlir::Attribute addrSpace = + mlir::DataLayout(module).getProgramMemorySpace()) + return llvm::cast(addrSpace).getUInt(); + return defaultAddressSpace; +} + +} // namespace fir -- GitLab From 19268ac55106834701b1c41acded4d413a3502e9 Mon Sep 17 00:00:00 2001 From: Mariusz Borsa Date: Fri, 22 Mar 2024 13:36:17 -0700 Subject: [PATCH 102/306] [Sanitizers][Darwin] Bump up DEFAULT_SANITIZER_MIN_OSX_VERSION (#86035) The greendragon was recently moved and now it runs on somewhat newer macOS version - which breaks some sanitizers tests rdar://125052915 Co-authored-by: Mariusz Borsa --- compiler-rt/cmake/config-ix.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler-rt/cmake/config-ix.cmake b/compiler-rt/cmake/config-ix.cmake index 4f47142850a5..46a6fdf8728f 100644 --- a/compiler-rt/cmake/config-ix.cmake +++ b/compiler-rt/cmake/config-ix.cmake @@ -461,7 +461,7 @@ if(APPLE) set(ORC_SUPPORTED_OS osx) endif() - set(DEFAULT_SANITIZER_MIN_OSX_VERSION 10.10) + set(DEFAULT_SANITIZER_MIN_OSX_VERSION 10.13) set(DARWIN_osx_MIN_VER_FLAG "-mmacosx-version-min") if(NOT SANITIZER_MIN_OSX_VERSION) string(REGEX MATCH "${DARWIN_osx_MIN_VER_FLAG}=([.0-9]+)" -- GitLab From 105feb9ac61243a32c84f5c13015602e5de500ff Mon Sep 17 00:00:00 2001 From: Alexander Yermolovich <43973793+ayermolo@users.noreply.github.com> Date: Fri, 22 Mar 2024 13:41:27 -0700 Subject: [PATCH 103/306] [BOLT][DWARF] Fix handling of DW_TAG_label (#86182) For DWARF5 BOLT was not retreiving address and instead was setting an index. Changed so that an address is used, and added DWARF4 test because it was missing. --- bolt/lib/Rewrite/DWARFRewriter.cpp | 43 +++-- bolt/test/X86/dwarf4-label-low-pc.s | 263 ++++++++++++++++++++++++++++ bolt/test/X86/dwarf5-label-low-pc.s | 36 ++-- 3 files changed, 316 insertions(+), 26 deletions(-) create mode 100644 bolt/test/X86/dwarf4-label-low-pc.s diff --git a/bolt/lib/Rewrite/DWARFRewriter.cpp b/bolt/lib/Rewrite/DWARFRewriter.cpp index 8dc7b90a6e30..601a2105fc26 100644 --- a/bolt/lib/Rewrite/DWARFRewriter.cpp +++ b/bolt/lib/Rewrite/DWARFRewriter.cpp @@ -375,12 +375,11 @@ static cl::opt AlwaysConvertToRanges( extern cl::opt CompDirOverride; } // namespace opts -static bool getLowAndHighPC(const DIE &Die, const DWARFUnit &DU, - uint64_t &LowPC, uint64_t &HighPC, - uint64_t &SectionIndex) { +/// If DW_AT_low_pc exists sets LowPC and returns true. +static bool getLowPC(const DIE &Die, const DWARFUnit &DU, uint64_t &LowPC, + uint64_t &SectionIndex) { DIEValue DvalLowPc = Die.findAttribute(dwarf::DW_AT_low_pc); - DIEValue DvalHighPc = Die.findAttribute(dwarf::DW_AT_high_pc); - if (!DvalLowPc || !DvalHighPc) + if (!DvalLowPc) return false; dwarf::Form Form = DvalLowPc.getForm(); @@ -403,14 +402,39 @@ static bool getLowAndHighPC(const DIE &Die, const DWARFUnit &DU, LowPC = LowPcValue; SectionIndex = 0; } + return true; +} + +/// If DW_AT_high_pc exists sets HighPC and returns true. +static bool getHighPC(const DIE &Die, const uint64_t LowPC, uint64_t &HighPC) { + DIEValue DvalHighPc = Die.findAttribute(dwarf::DW_AT_high_pc); + if (!DvalHighPc) + return false; if (DvalHighPc.getForm() == dwarf::DW_FORM_addr) HighPC = DvalHighPc.getDIEInteger().getValue(); else HighPC = LowPC + DvalHighPc.getDIEInteger().getValue(); - return true; } +/// If DW_AT_low_pc and DW_AT_high_pc exist sets LowPC and HighPC and returns +/// true. +static bool getLowAndHighPC(const DIE &Die, const DWARFUnit &DU, + uint64_t &LowPC, uint64_t &HighPC, + uint64_t &SectionIndex) { + uint64_t TempLowPC = LowPC; + uint64_t TempHighPC = HighPC; + uint64_t TempSectionIndex = SectionIndex; + if (getLowPC(Die, DU, TempLowPC, TempSectionIndex) && + getHighPC(Die, TempLowPC, TempHighPC)) { + LowPC = TempLowPC; + HighPC = TempHighPC; + SectionIndex = TempSectionIndex; + return true; + } + return false; +} + static Expected getDIEAddressRanges(const DIE &Die, DWARFUnit &DU) { uint64_t LowPC, HighPC, Index; @@ -1248,10 +1272,9 @@ void DWARFRewriter::updateUnitDebugInfo( } } } else if (LowPCAttrInfo) { - const std::optional Result = - LowPCAttrInfo.getDIEInteger().getValue(); - if (Result.has_value()) { - const uint64_t Address = Result.value(); + uint64_t Address = 0; + uint64_t SectionIndex = 0; + if (getLowPC(*Die, Unit, Address, SectionIndex)) { uint64_t NewAddress = 0; if (const BinaryFunction *Function = BC.getBinaryFunctionContainingAddress(Address)) { diff --git a/bolt/test/X86/dwarf4-label-low-pc.s b/bolt/test/X86/dwarf4-label-low-pc.s new file mode 100644 index 000000000000..dfd5af18c09b --- /dev/null +++ b/bolt/test/X86/dwarf4-label-low-pc.s @@ -0,0 +1,263 @@ + +# REQUIRES: system-linux + +# RUN: llvm-mc -dwarf-version=4 -filetype=obj -triple x86_64-unknown-linux %s -o %tmain.o +# RUN: %clang %cflags -dwarf-4 %tmain.o -o %t.exe -Wl,-q +# RUN: llvm-bolt %t.exe -o %t.bolt --update-debug-sections +# RUN: llvm-dwarfdump --show-form --verbose --debug-info %t.exe | FileCheck --check-prefix=PRECHECK %s +# RUN: llvm-dwarfdump --show-form --verbose --debug-info %t.bolt > %t.txt +# RUN: llvm-objdump -d %t.bolt >> %t.txt +# RUN: cat %t.txt | FileCheck --check-prefix=POSTCHECK %s + +## This test checks that we correctly handle DW_AT_low_pc [DW_FORM_addr] that is part of DW_TAG_label. + +# PRECHECK: version = 0x0004 +# PRECHECK: DW_TAG_label +# PRECHECK-NEXT: DW_AT_name +# PRECHECK-NEXT: DW_AT_decl_file +# PRECHECK-NEXT: DW_AT_decl_line +# PRECHECK-NEXT:DW_AT_low_pc [DW_FORM_addr] +# PRECHECK: DW_TAG_label +# PRECHECK-NEXT: DW_AT_name +# PRECHECK-NEXT: DW_AT_decl_file +# PRECHECK-NEXT: DW_AT_decl_line +# PRECHECK-NEXT:DW_AT_low_pc [DW_FORM_addr] + +# POSTCHECK: version = 0x0004 +# POSTCHECK: DW_TAG_label +# POSTCHECK-NEXT: DW_AT_name +# POSTCHECK-NEXT: DW_AT_decl_file +# POSTCHECK-NEXT: DW_AT_decl_line +# POSTCHECK-NEXT:DW_AT_low_pc [DW_FORM_addr] (0x[[ADDR:[1-9a-f]*]] +# POSTCHECK: DW_TAG_label +# POSTCHECK-NEXT: DW_AT_name +# POSTCHECK-NEXT: DW_AT_decl_file +# POSTCHECK-NEXT: DW_AT_decl_line +# POSTCHECK-NEXT:DW_AT_low_pc [DW_FORM_addr] (0x[[ADDR2:[1-9a-f]*]] + +# POSTCHECK: [[ADDR]]: 8b 45 f8 +# POSTCHECK: [[ADDR2]]: 8b 45 f8 + +## clang++ main.cpp -g2 -gdwarf-4 -S +## int main() { +## int a = 4; +## if (a == 5) +## goto LABEL1; +## else +## goto LABEL2; +## LABEL1:a++; +## LABEL2:a--; +## return 0; +## } + + .text + .file "main.cpp" + .globl main # -- Begin function main + .p2align 4, 0x90 + .type main,@function +main: # @main +.Lfunc_begin0: + .file 1 "/home" "main.cpp" + .loc 1 1 0 # main.cpp:1:0 + .cfi_startproc +# %bb.0: # %entry + pushq %rbp + .cfi_def_cfa_offset 16 + .cfi_offset %rbp, -16 + movq %rsp, %rbp + .cfi_def_cfa_register %rbp + movl $0, -4(%rbp) +.Ltmp0: + .loc 1 2 7 prologue_end # main.cpp:2:7 + movl $4, -8(%rbp) +.Ltmp1: + .loc 1 3 9 # main.cpp:3:9 + cmpl $5, -8(%rbp) +.Ltmp2: + .loc 1 3 7 is_stmt 0 # main.cpp:3:7 + jne .LBB0_2 +# %bb.1: # %if.then +.Ltmp3: + .loc 1 4 5 is_stmt 1 # main.cpp:4:5 + jmp .LBB0_3 +.LBB0_2: # %if.else + .loc 1 6 5 # main.cpp:6:5 + jmp .LBB0_4 +.Ltmp4: +.LBB0_3: # %LABEL1 + #DEBUG_LABEL: main:LABEL1 + .loc 1 7 11 # main.cpp:7:11 + movl -8(%rbp), %eax + addl $1, %eax + movl %eax, -8(%rbp) +.LBB0_4: # %LABEL2 +.Ltmp5: + #DEBUG_LABEL: main:LABEL2 + .loc 1 8 11 # main.cpp:8:11 + movl -8(%rbp), %eax + addl $-1, %eax + movl %eax, -8(%rbp) + .loc 1 9 3 # main.cpp:9:3 + xorl %eax, %eax + .loc 1 9 3 epilogue_begin is_stmt 0 # main.cpp:9:3 + popq %rbp + .cfi_def_cfa %rsp, 8 + retq +.Ltmp6: +.Lfunc_end0: + .size main, .Lfunc_end0-main + .cfi_endproc + # -- End function + .section .debug_abbrev,"",@progbits + .byte 1 # Abbreviation Code + .byte 17 # DW_TAG_compile_unit + .byte 1 # DW_CHILDREN_yes + .byte 37 # DW_AT_producer + .byte 14 # DW_FORM_strp + .byte 19 # DW_AT_language + .byte 5 # DW_FORM_data2 + .byte 3 # DW_AT_name + .byte 14 # DW_FORM_strp + .byte 16 # DW_AT_stmt_list + .byte 23 # DW_FORM_sec_offset + .byte 27 # DW_AT_comp_dir + .byte 14 # DW_FORM_strp + .byte 17 # DW_AT_low_pc + .byte 1 # DW_FORM_addr + .byte 18 # DW_AT_high_pc + .byte 6 # DW_FORM_data4 + .byte 0 # EOM(1) + .byte 0 # EOM(2) + .byte 2 # Abbreviation Code + .byte 46 # DW_TAG_subprogram + .byte 1 # DW_CHILDREN_yes + .byte 17 # DW_AT_low_pc + .byte 1 # DW_FORM_addr + .byte 18 # DW_AT_high_pc + .byte 6 # DW_FORM_data4 + .byte 64 # DW_AT_frame_base + .byte 24 # DW_FORM_exprloc + .byte 3 # DW_AT_name + .byte 14 # DW_FORM_strp + .byte 58 # DW_AT_decl_file + .byte 11 # DW_FORM_data1 + .byte 59 # DW_AT_decl_line + .byte 11 # DW_FORM_data1 + .byte 73 # DW_AT_type + .byte 19 # DW_FORM_ref4 + .byte 63 # DW_AT_external + .byte 25 # DW_FORM_flag_present + .byte 0 # EOM(1) + .byte 0 # EOM(2) + .byte 3 # Abbreviation Code + .byte 52 # DW_TAG_variable + .byte 0 # DW_CHILDREN_no + .byte 2 # DW_AT_location + .byte 24 # DW_FORM_exprloc + .byte 3 # DW_AT_name + .byte 14 # DW_FORM_strp + .byte 58 # DW_AT_decl_file + .byte 11 # DW_FORM_data1 + .byte 59 # DW_AT_decl_line + .byte 11 # DW_FORM_data1 + .byte 73 # DW_AT_type + .byte 19 # DW_FORM_ref4 + .byte 0 # EOM(1) + .byte 0 # EOM(2) + .byte 4 # Abbreviation Code + .byte 10 # DW_TAG_label + .byte 0 # DW_CHILDREN_no + .byte 3 # DW_AT_name + .byte 14 # DW_FORM_strp + .byte 58 # DW_AT_decl_file + .byte 11 # DW_FORM_data1 + .byte 59 # DW_AT_decl_line + .byte 11 # DW_FORM_data1 + .byte 17 # DW_AT_low_pc + .byte 1 # DW_FORM_addr + .byte 0 # EOM(1) + .byte 0 # EOM(2) + .byte 5 # Abbreviation Code + .byte 36 # DW_TAG_base_type + .byte 0 # DW_CHILDREN_no + .byte 3 # DW_AT_name + .byte 14 # DW_FORM_strp + .byte 62 # DW_AT_encoding + .byte 11 # DW_FORM_data1 + .byte 11 # DW_AT_byte_size + .byte 11 # DW_FORM_data1 + .byte 0 # EOM(1) + .byte 0 # EOM(2) + .byte 0 # EOM(3) + .section .debug_info,"",@progbits +.Lcu_begin0: + .long .Ldebug_info_end0-.Ldebug_info_start0 # Length of Unit +.Ldebug_info_start0: + .short 4 # DWARF version number + .long .debug_abbrev # Offset Into Abbrev. Section + .byte 8 # Address Size (in bytes) + .byte 1 # Abbrev [1] 0xb:0x6d DW_TAG_compile_unit + .long .Linfo_string0 # DW_AT_producer + .short 33 # DW_AT_language + .long .Linfo_string1 # DW_AT_name + .long .Lline_table_start0 # DW_AT_stmt_list + .long .Linfo_string2 # DW_AT_comp_dir + .quad .Lfunc_begin0 # DW_AT_low_pc + .long .Lfunc_end0-.Lfunc_begin0 # DW_AT_high_pc + .byte 2 # Abbrev [2] 0x2a:0x46 DW_TAG_subprogram + .quad .Lfunc_begin0 # DW_AT_low_pc + .long .Lfunc_end0-.Lfunc_begin0 # DW_AT_high_pc + .byte 1 # DW_AT_frame_base + .byte 86 + .long .Linfo_string3 # DW_AT_name + .byte 1 # DW_AT_decl_file + .byte 1 # DW_AT_decl_line + .long 112 # DW_AT_type + # DW_AT_external + .byte 3 # Abbrev [3] 0x43:0xe DW_TAG_variable + .byte 2 # DW_AT_location + .byte 145 + .byte 120 + .long .Linfo_string5 # DW_AT_name + .byte 1 # DW_AT_decl_file + .byte 2 # DW_AT_decl_line + .long 112 # DW_AT_type + .byte 4 # Abbrev [4] 0x51:0xf DW_TAG_label + .long .Linfo_string6 # DW_AT_name + .byte 1 # DW_AT_decl_file + .byte 7 # DW_AT_decl_line + .quad .Ltmp4 # DW_AT_low_pc + .byte 4 # Abbrev [4] 0x60:0xf DW_TAG_label + .long .Linfo_string7 # DW_AT_name + .byte 1 # DW_AT_decl_file + .byte 8 # DW_AT_decl_line + .quad .Ltmp5 # DW_AT_low_pc + .byte 0 # End Of Children Mark + .byte 5 # Abbrev [5] 0x70:0x7 DW_TAG_base_type + .long .Linfo_string4 # DW_AT_name + .byte 5 # DW_AT_encoding + .byte 4 # DW_AT_byte_size + .byte 0 # End Of Children Mark +.Ldebug_info_end0: + .section .debug_str,"MS",@progbits,1 +.Linfo_string0: + .asciz "clang version 19.0.0git" # string offset=0 +.Linfo_string1: + .asciz "main.cpp" # string offset=24 +.Linfo_string2: + .asciz "/home" # string offset=33 +.Linfo_string3: + .asciz "main" # string offset=71 +.Linfo_string4: + .asciz "int" # string offset=76 +.Linfo_string5: + .asciz "a" # string offset=80 +.Linfo_string6: + .asciz "LABEL1" # string offset=82 +.Linfo_string7: + .asciz "LABEL2" # string offset=89 + .ident "clang version 19.0.0git" + .section ".note.GNU-stack","",@progbits + .addrsig + .section .debug_line,"",@progbits +.Lline_table_start0: diff --git a/bolt/test/X86/dwarf5-label-low-pc.s b/bolt/test/X86/dwarf5-label-low-pc.s index 890d9e024d1a..1e3fc17ad516 100644 --- a/bolt/test/X86/dwarf5-label-low-pc.s +++ b/bolt/test/X86/dwarf5-label-low-pc.s @@ -8,9 +8,10 @@ # RUN: llvm-dwarfdump --show-form --verbose --debug-addr %t.bolt > %t.txt # RUN: llvm-dwarfdump --show-form --verbose --debug-info %t.bolt >> %t.txt +# RUN: llvm-objdump -d %t.bolt >> %t.txt # RUN: cat %t.txt | FileCheck --check-prefix=POSTCHECK %s -# This test checks that we correctly handle DW_AT_low_pc [DW_FORM_addrx] that is part of DW_TAG_label. +## This test checks that we correctly handle DW_AT_low_pc [DW_FORM_addrx] that is part of DW_TAG_label. # PRECHECK: version = 0x0005 # PRECHECK: DW_TAG_label @@ -28,8 +29,8 @@ # POSTCHECK: Addrs: [ # POSTCHECK-NEXT: 0x # POSTCHECK-NEXT: 0x -# POSTCHECK-NEXT: 0x[[#%.16x,ADDR:]] -# POSTCHECK-NEXT: 0x[[#%.16x,ADDR2:]] +# POSTCHECK-NEXT: 0x[[ADDR:[1-9a-f]*]] +# POSTCHECK-NEXT: 0x[[ADDR2:[1-9a-f]*]] # POSTCHECK: version = 0x0005 # POSTCHECK: DW_TAG_label @@ -37,25 +38,28 @@ # POSTCHECK-NEXT: DW_AT_decl_file # POSTCHECK-NEXT: DW_AT_decl_line # POSTCHECK-NEXT:DW_AT_low_pc [DW_FORM_addrx] (indexed (00000002) -# POSTCHECK-SAME: 0x[[#ADDR]] +# POSTCHECK-SAME: 0x[[ADDR]] # POSTCHECK: DW_TAG_label # POSTCHECK-NEXT: DW_AT_name # POSTCHECK-NEXT: DW_AT_decl_file # POSTCHECK-NEXT: DW_AT_decl_line # POSTCHECK-NEXT:DW_AT_low_pc [DW_FORM_addrx] (indexed (00000003) -# POSTCHECK-SAME: 0x[[#ADDR2]] +# POSTCHECK-SAME: 0x[[ADDR2]] -# clang++ main.cpp -g -S -# int main() { -# int a = 4; -# if (a == 5) -# goto LABEL1; -# else -# goto LABEL2; -# LABEL1:a++; -# LABEL2:a--; -# return 0; -# } +# POSTCHECK: [[ADDR]]: 8b 45 f8 +# POSTCHECK: [[ADDR2]]: 8b 45 f8 + +## clang++ main.cpp -g -S +## int main() { +## int a = 4; +## if (a == 5) +## goto LABEL1; +## else +## goto LABEL2; +## LABEL1:a++; +## LABEL2:a--; +## return 0; +## } .text .file "main.cpp" -- GitLab From f3cfe016c5d8429c0dccfa6f85442e2ea0d45a58 Mon Sep 17 00:00:00 2001 From: Alexander Yermolovich <43973793+ayermolo@users.noreply.github.com> Date: Fri, 22 Mar 2024 13:48:49 -0700 Subject: [PATCH 104/306] [BOLT][DWARF] Add support for cross-cu references for debug-names (#86015) The DW_AT_abstract_origin can be a cross-cu reference as a by-product of LTO. On IR level for absolute references an address is stored, vs a DIE for relative references. Added a map to keep track of cross-cu referenced DIEs to use when we add an Entry. --- bolt/include/bolt/Core/DebugNames.h | 11 + bolt/lib/Core/DIEBuilder.cpp | 6 +- bolt/lib/Core/DebugNames.cpp | 109 +-- bolt/test/X86/dwarf5-debug-names-cross-cu.s | 712 ++++++++++++++++++++ 4 files changed, 796 insertions(+), 42 deletions(-) create mode 100644 bolt/test/X86/dwarf5-debug-names-cross-cu.s diff --git a/bolt/include/bolt/Core/DebugNames.h b/bolt/include/bolt/Core/DebugNames.h index 1f17f1ae4d13..fbaa7f4e68aa 100644 --- a/bolt/include/bolt/Core/DebugNames.h +++ b/bolt/include/bolt/Core/DebugNames.h @@ -68,6 +68,16 @@ public: std::unique_ptr releaseBuffer() { return std::move(FullTableBuffer); } + /// Adds a DIE that is referenced across CUs. + void addCrossCUDie(const DIE *Die) { + CrossCUDies.insert({Die->getOffset(), Die}); + } + /// Returns true if the DIE can generate an entry for a cross cu reference. + /// This only checks TAGs of a DIE because when this is invoked DIE might not + /// be fully constructed. + bool canGenerateEntryWithCrossCUReference( + const DWARFUnit &Unit, const DIE &Die, + const DWARFAbbreviationDeclaration::AttributeSpec &AttrSpec); private: BinaryContext &BC; @@ -128,6 +138,7 @@ private: llvm::DenseMap CUOffsetsToPatch; // Contains a map of Entry ID to Entry relative offset. llvm::DenseMap EntryRelativeOffsets; + llvm::DenseMap CrossCUDies; /// Adds Unit to either CUList, LocalTUList or ForeignTUList. /// Input Unit being processed, and DWO ID if Unit is being processed comes /// from a DWO section. diff --git a/bolt/lib/Core/DIEBuilder.cpp b/bolt/lib/Core/DIEBuilder.cpp index 0cf8a5e8c2c3..354fe5059443 100644 --- a/bolt/lib/Core/DIEBuilder.cpp +++ b/bolt/lib/Core/DIEBuilder.cpp @@ -545,6 +545,10 @@ void DIEBuilder::cloneDieReferenceAttribute( NewRefDie = DieInfo.Die; if (AttrSpec.Form == dwarf::DW_FORM_ref_addr) { + // Adding referenced DIE to DebugNames to be used when entries are created + // that contain cross cu references. + if (DebugNamesTable.canGenerateEntryWithCrossCUReference(U, Die, AttrSpec)) + DebugNamesTable.addCrossCUDie(DieInfo.Die); // no matter forward reference or backward reference, we are supposed // to calculate them in `finish` due to the possible modification of // the DIE. @@ -554,7 +558,7 @@ void DIEBuilder::cloneDieReferenceAttribute( std::make_pair(CurDieInfo, AddrReferenceInfo(&DieInfo, AttrSpec))); Die.addValue(getState().DIEAlloc, AttrSpec.Attr, dwarf::DW_FORM_ref_addr, - DIEInteger(0xDEADBEEF)); + DIEInteger(DieInfo.Die->getOffset())); return; } diff --git a/bolt/lib/Core/DebugNames.cpp b/bolt/lib/Core/DebugNames.cpp index 23a29f52513c..049244c4b515 100644 --- a/bolt/lib/Core/DebugNames.cpp +++ b/bolt/lib/Core/DebugNames.cpp @@ -146,6 +146,55 @@ static bool shouldIncludeVariable(const DWARFUnit &Unit, const DIE &Die) { return false; } +bool static canProcess(const DWARFUnit &Unit, const DIE &Die, + std::string &NameToUse, const bool TagsOnly) { + switch (Die.getTag()) { + case dwarf::DW_TAG_base_type: + case dwarf::DW_TAG_class_type: + case dwarf::DW_TAG_enumeration_type: + case dwarf::DW_TAG_imported_declaration: + case dwarf::DW_TAG_pointer_type: + case dwarf::DW_TAG_structure_type: + case dwarf::DW_TAG_typedef: + case dwarf::DW_TAG_unspecified_type: + if (TagsOnly || Die.findAttribute(dwarf::Attribute::DW_AT_name)) + return true; + return false; + case dwarf::DW_TAG_namespace: + // According to DWARF5 spec namespaces without DW_AT_name needs to have + // "(anonymous namespace)" + if (!Die.findAttribute(dwarf::Attribute::DW_AT_name)) + NameToUse = "(anonymous namespace)"; + return true; + case dwarf::DW_TAG_inlined_subroutine: + case dwarf::DW_TAG_label: + case dwarf::DW_TAG_subprogram: + if (TagsOnly || Die.findAttribute(dwarf::Attribute::DW_AT_low_pc) || + Die.findAttribute(dwarf::Attribute::DW_AT_high_pc) || + Die.findAttribute(dwarf::Attribute::DW_AT_ranges) || + Die.findAttribute(dwarf::Attribute::DW_AT_entry_pc)) + return true; + return false; + case dwarf::DW_TAG_variable: + return TagsOnly || shouldIncludeVariable(Unit, Die); + default: + break; + } + return false; +} + +bool DWARF5AcceleratorTable::canGenerateEntryWithCrossCUReference( + const DWARFUnit &Unit, const DIE &Die, + const DWARFAbbreviationDeclaration::AttributeSpec &AttrSpec) { + if (!isCreated()) + return false; + std::string NameToUse = ""; + if (!canProcess(Unit, Die, NameToUse, true)) + return false; + return (AttrSpec.Attr == dwarf::Attribute::DW_AT_abstract_origin || + AttrSpec.Attr == dwarf::Attribute::DW_AT_specification) && + AttrSpec.Form == dwarf::DW_FORM_ref_addr; +} /// Returns name offset in String Offset section. static uint64_t getNameOffset(BinaryContext &BC, DWARFUnit &Unit, const uint64_t Index) { @@ -175,41 +224,6 @@ DWARF5AcceleratorTable::addAccelTableEntry( if (Unit.getVersion() < 5 || !NeedToCreate) return std::nullopt; std::string NameToUse = ""; - auto canProcess = [&](const DIE &Die) -> bool { - switch (Die.getTag()) { - case dwarf::DW_TAG_base_type: - case dwarf::DW_TAG_class_type: - case dwarf::DW_TAG_enumeration_type: - case dwarf::DW_TAG_imported_declaration: - case dwarf::DW_TAG_pointer_type: - case dwarf::DW_TAG_structure_type: - case dwarf::DW_TAG_typedef: - case dwarf::DW_TAG_unspecified_type: - if (Die.findAttribute(dwarf::Attribute::DW_AT_name)) - return true; - return false; - case dwarf::DW_TAG_namespace: - // According to DWARF5 spec namespaces without DW_AT_name needs to have - // "(anonymous namespace)" - if (!Die.findAttribute(dwarf::Attribute::DW_AT_name)) - NameToUse = "(anonymous namespace)"; - return true; - case dwarf::DW_TAG_inlined_subroutine: - case dwarf::DW_TAG_label: - case dwarf::DW_TAG_subprogram: - if (Die.findAttribute(dwarf::Attribute::DW_AT_low_pc) || - Die.findAttribute(dwarf::Attribute::DW_AT_high_pc) || - Die.findAttribute(dwarf::Attribute::DW_AT_ranges) || - Die.findAttribute(dwarf::Attribute::DW_AT_entry_pc)) - return true; - return false; - case dwarf::DW_TAG_variable: - return shouldIncludeVariable(Unit, Die); - default: - break; - } - return false; - }; auto getUnitID = [&](const DWARFUnit &Unit, bool &IsTU, uint32_t &DieTag) -> uint32_t { @@ -223,7 +237,7 @@ DWARF5AcceleratorTable::addAccelTableEntry( return CUList.size() - 1; }; - if (!canProcess(Die)) + if (!canProcess(Unit, Die, NameToUse, false)) return std::nullopt; // Addes a Unit to either CU, LocalTU or ForeignTU list the first time we @@ -318,10 +332,24 @@ DWARF5AcceleratorTable::addAccelTableEntry( const DIEValue Value = Die.findAttribute(Attr); if (!Value) return std::nullopt; - const DIEEntry &DIEENtry = Value.getDIEEntry(); - DIE &EntryDie = DIEENtry.getEntry(); - addEntry(EntryDie.findAttribute(dwarf::Attribute::DW_AT_linkage_name)); - return addEntry(EntryDie.findAttribute(dwarf::Attribute::DW_AT_name)); + const DIE *EntryDie = nullptr; + if (Value.getForm() == dwarf::DW_FORM_ref_addr) { + auto Iter = CrossCUDies.find(Value.getDIEInteger().getValue()); + if (Iter == CrossCUDies.end()) { + BC.errs() << "BOLT-WARNING: [internal-dwarf-warning]: Could not find " + "referenced DIE in CrossCUDies for " + << Twine::utohexstr(Value.getDIEInteger().getValue()) + << ".\n"; + return std::nullopt; + } + EntryDie = Iter->second; + } else { + const DIEEntry &DIEENtry = Value.getDIEEntry(); + EntryDie = &DIEENtry.getEntry(); + } + + addEntry(EntryDie->findAttribute(dwarf::Attribute::DW_AT_linkage_name)); + return addEntry(EntryDie->findAttribute(dwarf::Attribute::DW_AT_name)); }; if (std::optional Entry = @@ -332,7 +360,6 @@ DWARF5AcceleratorTable::addAccelTableEntry( return *Entry; return addEntry(Die.findAttribute(dwarf::Attribute::DW_AT_name)); - ; } /// Algorithm from llvm implementation. diff --git a/bolt/test/X86/dwarf5-debug-names-cross-cu.s b/bolt/test/X86/dwarf5-debug-names-cross-cu.s new file mode 100644 index 000000000000..73c50d6d41db --- /dev/null +++ b/bolt/test/X86/dwarf5-debug-names-cross-cu.s @@ -0,0 +1,712 @@ + +# REQUIRES: system-linux + +# RUN: llvm-mc -dwarf-version=5 -filetype=obj -triple x86_64-unknown-linux %s -o %tmain.o +# RUN: %clang %cflags -dwarf-5 %tmain.o -o %t.exe -Wl,-q +# RUN: llvm-bolt %t.exe -o %t.bolt --update-debug-sections +# RUN: llvm-dwarfdump --debug-info -r 0 --debug-names %t.bolt > %t.txt +# RUN: cat %t.txt | FileCheck --check-prefix=CHECK %s + +## This test checks that BOLT generates Entries for DW_AT_abstract_origin when it has cross cu reference. + +# CHECK: [[OFFSET1:0x[0-9a-f]*]]: Compile Unit +# CHECK: [[OFFSET2:0x[0-9a-f]*]]: Compile Unit +# CHECK: Name Index @ 0x0 { +# CHECK-NEXT: Header { +# CHECK-NEXT: Length: 0xD2 +# CHECK-NEXT: Format: DWARF32 +# CHECK-NEXT: Version: 5 +# CHECK-NEXT: CU count: 2 +# CHECK-NEXT: Local TU count: 0 +# CHECK-NEXT: Foreign TU count: 0 +# CHECK-NEXT: Bucket count: 5 +# CHECK-NEXT: Name count: 5 +# CHECK-NEXT: Abbreviations table size: 0x1F +# CHECK-NEXT: Augmentation: 'BOLT' +# CHECK-NEXT: } +# CHECK-NEXT: Compilation Unit offsets [ +# CHECK-NEXT: CU[0]: [[OFFSET1]] +# CHECK-NEXT: CU[1]: [[OFFSET2]] +# CHECK-NEXT: ] +# CHECK-NEXT: Abbreviations [ +# CHECK-NEXT: Abbreviation [[ABBREV1:0x[0-9a-f]*]] { +# CHECK-NEXT: Tag: DW_TAG_subprogram +# CHECK-NEXT: DW_IDX_compile_unit: DW_FORM_data1 +# CHECK-NEXT: DW_IDX_die_offset: DW_FORM_ref4 +# CHECK-NEXT: DW_IDX_parent: DW_FORM_flag_present +# CHECK-NEXT: } +# CHECK-NEXT: Abbreviation [[ABBREV2:0x[0-9a-f]*]] { +# CHECK-NEXT: Tag: DW_TAG_inlined_subroutine +# CHECK-NEXT: DW_IDX_compile_unit: DW_FORM_data1 +# CHECK-NEXT: DW_IDX_die_offset: DW_FORM_ref4 +# CHECK-NEXT: DW_IDX_parent: DW_FORM_ref4 +# CHECK-NEXT: } +# CHECK-NEXT: Abbreviation [[ABBREV3:0x[0-9a-f]*]] { +# CHECK-NEXT: Tag: DW_TAG_base_type +# CHECK-NEXT: DW_IDX_compile_unit: DW_FORM_data1 +# CHECK-NEXT: DW_IDX_die_offset: DW_FORM_ref4 +# CHECK-NEXT: DW_IDX_parent: DW_FORM_flag_present +# CHECK-NEXT: } +# CHECK-NEXT: ] +# CHECK-NEXT: Bucket 0 [ +# CHECK-NEXT: EMPTY +# CHECK-NEXT: ] +# CHECK-NEXT: Bucket 1 [ +# CHECK-NEXT: Name 1 { +# CHECK-NEXT: Hash: 0x7C9A7F6A +# CHECK-NEXT: String: {{.+}} "main" +# CHECK-NEXT: Entry @ [[ENTRY:0x[0-9a-f]*]] { +# CHECK-NEXT: Abbrev: [[ABBREV1]] +# CHECK-NEXT: Tag: DW_TAG_subprogram +# CHECK-NEXT: DW_IDX_compile_unit: 0x00 +# CHECK-NEXT: DW_IDX_die_offset: 0x00000024 +# CHECK-NEXT: DW_IDX_parent: +# CHECK-NEXT: } +# CHECK-NEXT: } +# CHECK-NEXT: Name 2 { +# CHECK-NEXT: Hash: 0xB5063CFE +# CHECK-NEXT: String: {{.+}} "_Z3fooi" +# CHECK-NEXT: Entry @ {{.+}} { +# CHECK-NEXT: Abbrev: [[ABBREV1]] +# CHECK-NEXT: Tag: DW_TAG_subprogram +# CHECK-NEXT: DW_IDX_compile_unit: 0x01 +# CHECK-NEXT: DW_IDX_die_offset: 0x0000003a +# CHECK-NEXT: DW_IDX_parent: +# CHECK-NEXT: } +# CHECK-NEXT: Entry @ {{.+}} { +# CHECK-NEXT: Abbrev: [[ABBREV2]] +# CHECK-NEXT: Tag: DW_TAG_inlined_subroutine +# CHECK-NEXT: DW_IDX_compile_unit: 0x00 +# CHECK-NEXT: DW_IDX_die_offset: 0x00000054 +# CHECK-NEXT: DW_IDX_parent: Entry @ [[ENTRY]] +# CHECK-NEXT: } +# CHECK-NEXT: } +# CHECK-NEXT: ] +# CHECK-NEXT: Bucket 2 [ +# CHECK-NEXT: EMPTY +# CHECK-NEXT: ] +# CHECK-NEXT: Bucket 3 [ +# CHECK-NEXT: Name 3 { +# CHECK-NEXT: Hash: 0xB888030 +# CHECK-NEXT: String: {{.+}} "int" +# CHECK-NEXT: Entry @ {{.+}} { +# CHECK-NEXT: Abbrev: [[ABBREV3]] +# CHECK-NEXT: Tag: DW_TAG_base_type +# CHECK-NEXT: DW_IDX_compile_unit: 0x01 +# CHECK-NEXT: DW_IDX_die_offset: 0x00000036 +# CHECK-NEXT: DW_IDX_parent: +# CHECK-NEXT: } +# CHECK-NEXT: } +# CHECK-NEXT: ] +# CHECK-NEXT: Bucket 4 [ +# CHECK-NEXT: Name 4 { +# CHECK-NEXT: Hash: 0xB887389 +# CHECK-NEXT: String: {{.+}} "foo" +# CHECK-NEXT: Entry @ {{.+}} { +# CHECK-NEXT: Abbrev: [[ABBREV1]] +# CHECK-NEXT: Tag: DW_TAG_subprogram +# CHECK-NEXT: DW_IDX_compile_unit: 0x01 +# CHECK-NEXT: DW_IDX_die_offset: 0x0000003a +# CHECK-NEXT: DW_IDX_parent: +# CHECK-NEXT: } +# CHECK-NEXT: Entry @ 0xc4 { +# CHECK-NEXT: Abbrev: [[ABBREV2]] +# CHECK-NEXT: Tag: DW_TAG_inlined_subroutine +# CHECK-NEXT: DW_IDX_compile_unit: 0x00 +# CHECK-NEXT: DW_IDX_die_offset: 0x00000054 +# CHECK-NEXT: DW_IDX_parent: Entry @ [[ENTRY]] +# CHECK-NEXT: } +# CHECK-NEXT: } +# CHECK-NEXT: Name 5 { +# CHECK-NEXT: Hash: 0x7C952063 +# CHECK-NEXT: String: {{.+}} "char" +# CHECK-NEXT: Entry @ {{.+}} { +# CHECK-NEXT: Abbrev: [[ABBREV3]] +# CHECK-NEXT: Tag: DW_TAG_base_type +# CHECK-NEXT: DW_IDX_compile_unit: 0x00 +# CHECK-NEXT: DW_IDX_die_offset: 0x00000075 +# CHECK-NEXT: DW_IDX_parent: +# CHECK-NEXT: } +# CHECK-NEXT: } +# CHECK-NEXT: ] +# CHECK-NEXT: } + +## clang++ -g2 -gpubnames -S -emit-llvm main.cpp -o main.ll +## clang++ -g2 -gpubnames -S -emit-llvm helper.cpp -o helper.ll +## llvm-link main.ll helper.ll -o combined.ll +## clang++ -g2 -gpubnames combined.ll -emit-llvm -S -o combined.opt.ll +## llc -dwarf-version=5 -filetype=asm -mtriple x86_64-unknown-linux combined.opt.ll -o combined.s +## main.cpp +## extern int foo(int); +## int main(int argc, char* argv[]) { +## int i = 0; +## [[clang::always_inline]] i = foo(argc); +## return i; +## } +## helper.cpp +## int foo(int i) { +## return i ++; +## } + + .text + .file "llvm-link" + .globl main # -- Begin function main + .p2align 4, 0x90 + .type main,@function +main: # @main +.Lfunc_begin0: + .file 1 "/home" "main.cpp" md5 0x24fb0b4c3900e91fece1ac87ed73ff3b + .loc 1 2 0 # main.cpp:2:0 + .cfi_startproc +# %bb.0: # %entry + pushq %rbp + .cfi_def_cfa_offset 16 + .cfi_offset %rbp, -16 + movq %rsp, %rbp + .cfi_def_cfa_register %rbp + movl $0, -16(%rbp) + movl %edi, -12(%rbp) + movq %rsi, -24(%rbp) +.Ltmp0: + .loc 1 3 7 prologue_end # main.cpp:3:7 + movl $0, -4(%rbp) + .loc 1 4 36 # main.cpp:4:36 + movl -12(%rbp), %eax + movl %eax, -8(%rbp) +.Ltmp1: + .file 2 "/home" "helper.cpp" md5 0x7d4429e24d8c74d7ee22c1889ad46d6b + .loc 2 2 12 # helper.cpp:2:12 + movl -8(%rbp), %eax + movl %eax, %ecx + addl $1, %ecx + movl %ecx, -8(%rbp) +.Ltmp2: + .loc 1 4 30 # main.cpp:4:30 + movl %eax, -4(%rbp) + .loc 1 5 10 # main.cpp:5:10 + movl -4(%rbp), %eax + .loc 1 5 3 epilogue_begin is_stmt 0 # main.cpp:5:3 + popq %rbp + .cfi_def_cfa %rsp, 8 + retq +.Ltmp3: +.Lfunc_end0: + .size main, .Lfunc_end0-main + .cfi_endproc + # -- End function + .globl _Z3fooi # -- Begin function _Z3fooi + .p2align 4, 0x90 + .type _Z3fooi,@function +_Z3fooi: # @_Z3fooi +.Lfunc_begin1: + .loc 2 1 0 is_stmt 1 # helper.cpp:1:0 + .cfi_startproc +# %bb.0: # %entry + pushq %rbp + .cfi_def_cfa_offset 16 + .cfi_offset %rbp, -16 + movq %rsp, %rbp + .cfi_def_cfa_register %rbp + movl %edi, -4(%rbp) +.Ltmp4: + .loc 2 2 12 prologue_end # helper.cpp:2:12 + movl -4(%rbp), %eax + movl %eax, %ecx + addl $1, %ecx + movl %ecx, -4(%rbp) + .loc 2 2 3 epilogue_begin is_stmt 0 # helper.cpp:2:3 + popq %rbp + .cfi_def_cfa %rsp, 8 + retq +.Ltmp5: +.Lfunc_end1: + .size _Z3fooi, .Lfunc_end1-_Z3fooi + .cfi_endproc + # -- End function + .section .debug_abbrev,"",@progbits + .byte 1 # Abbreviation Code + .byte 17 # DW_TAG_compile_unit + .byte 1 # DW_CHILDREN_yes + .byte 37 # DW_AT_producer + .byte 37 # DW_FORM_strx1 + .byte 19 # DW_AT_language + .byte 5 # DW_FORM_data2 + .byte 3 # DW_AT_name + .byte 37 # DW_FORM_strx1 + .byte 114 # DW_AT_str_offsets_base + .byte 23 # DW_FORM_sec_offset + .byte 16 # DW_AT_stmt_list + .byte 23 # DW_FORM_sec_offset + .byte 27 # DW_AT_comp_dir + .byte 37 # DW_FORM_strx1 + .byte 17 # DW_AT_low_pc + .byte 27 # DW_FORM_addrx + .byte 18 # DW_AT_high_pc + .byte 6 # DW_FORM_data4 + .byte 115 # DW_AT_addr_base + .byte 23 # DW_FORM_sec_offset + .byte 0 # EOM(1) + .byte 0 # EOM(2) + .byte 2 # Abbreviation Code + .byte 46 # DW_TAG_subprogram + .byte 1 # DW_CHILDREN_yes + .byte 17 # DW_AT_low_pc + .byte 27 # DW_FORM_addrx + .byte 18 # DW_AT_high_pc + .byte 6 # DW_FORM_data4 + .byte 64 # DW_AT_frame_base + .byte 24 # DW_FORM_exprloc + .byte 3 # DW_AT_name + .byte 37 # DW_FORM_strx1 + .byte 58 # DW_AT_decl_file + .byte 11 # DW_FORM_data1 + .byte 59 # DW_AT_decl_line + .byte 11 # DW_FORM_data1 + .byte 73 # DW_AT_type + .byte 16 # DW_FORM_ref_addr + .byte 63 # DW_AT_external + .byte 25 # DW_FORM_flag_present + .byte 0 # EOM(1) + .byte 0 # EOM(2) + .byte 3 # Abbreviation Code + .byte 5 # DW_TAG_formal_parameter + .byte 0 # DW_CHILDREN_no + .byte 2 # DW_AT_location + .byte 24 # DW_FORM_exprloc + .byte 3 # DW_AT_name + .byte 37 # DW_FORM_strx1 + .byte 58 # DW_AT_decl_file + .byte 11 # DW_FORM_data1 + .byte 59 # DW_AT_decl_line + .byte 11 # DW_FORM_data1 + .byte 73 # DW_AT_type + .byte 16 # DW_FORM_ref_addr + .byte 0 # EOM(1) + .byte 0 # EOM(2) + .byte 4 # Abbreviation Code + .byte 5 # DW_TAG_formal_parameter + .byte 0 # DW_CHILDREN_no + .byte 2 # DW_AT_location + .byte 24 # DW_FORM_exprloc + .byte 3 # DW_AT_name + .byte 37 # DW_FORM_strx1 + .byte 58 # DW_AT_decl_file + .byte 11 # DW_FORM_data1 + .byte 59 # DW_AT_decl_line + .byte 11 # DW_FORM_data1 + .byte 73 # DW_AT_type + .byte 19 # DW_FORM_ref4 + .byte 0 # EOM(1) + .byte 0 # EOM(2) + .byte 5 # Abbreviation Code + .byte 52 # DW_TAG_variable + .byte 0 # DW_CHILDREN_no + .byte 2 # DW_AT_location + .byte 24 # DW_FORM_exprloc + .byte 3 # DW_AT_name + .byte 37 # DW_FORM_strx1 + .byte 58 # DW_AT_decl_file + .byte 11 # DW_FORM_data1 + .byte 59 # DW_AT_decl_line + .byte 11 # DW_FORM_data1 + .byte 73 # DW_AT_type + .byte 16 # DW_FORM_ref_addr + .byte 0 # EOM(1) + .byte 0 # EOM(2) + .byte 6 # Abbreviation Code + .byte 29 # DW_TAG_inlined_subroutine + .byte 1 # DW_CHILDREN_yes + .byte 49 # DW_AT_abstract_origin + .byte 16 # DW_FORM_ref_addr + .byte 17 # DW_AT_low_pc + .byte 27 # DW_FORM_addrx + .byte 18 # DW_AT_high_pc + .byte 6 # DW_FORM_data4 + .byte 88 # DW_AT_call_file + .byte 11 # DW_FORM_data1 + .byte 89 # DW_AT_call_line + .byte 11 # DW_FORM_data1 + .byte 87 # DW_AT_call_column + .byte 11 # DW_FORM_data1 + .byte 0 # EOM(1) + .byte 0 # EOM(2) + .byte 7 # Abbreviation Code + .byte 5 # DW_TAG_formal_parameter + .byte 0 # DW_CHILDREN_no + .byte 2 # DW_AT_location + .byte 24 # DW_FORM_exprloc + .byte 49 # DW_AT_abstract_origin + .byte 16 # DW_FORM_ref_addr + .byte 0 # EOM(1) + .byte 0 # EOM(2) + .byte 8 # Abbreviation Code + .byte 15 # DW_TAG_pointer_type + .byte 0 # DW_CHILDREN_no + .byte 73 # DW_AT_type + .byte 19 # DW_FORM_ref4 + .byte 0 # EOM(1) + .byte 0 # EOM(2) + .byte 9 # Abbreviation Code + .byte 36 # DW_TAG_base_type + .byte 0 # DW_CHILDREN_no + .byte 3 # DW_AT_name + .byte 37 # DW_FORM_strx1 + .byte 62 # DW_AT_encoding + .byte 11 # DW_FORM_data1 + .byte 11 # DW_AT_byte_size + .byte 11 # DW_FORM_data1 + .byte 0 # EOM(1) + .byte 0 # EOM(2) + .byte 10 # Abbreviation Code + .byte 46 # DW_TAG_subprogram + .byte 1 # DW_CHILDREN_yes + .byte 110 # DW_AT_linkage_name + .byte 37 # DW_FORM_strx1 + .byte 3 # DW_AT_name + .byte 37 # DW_FORM_strx1 + .byte 58 # DW_AT_decl_file + .byte 11 # DW_FORM_data1 + .byte 59 # DW_AT_decl_line + .byte 11 # DW_FORM_data1 + .byte 73 # DW_AT_type + .byte 19 # DW_FORM_ref4 + .byte 63 # DW_AT_external + .byte 25 # DW_FORM_flag_present + .byte 32 # DW_AT_inline + .byte 33 # DW_FORM_implicit_const + .byte 1 + .byte 0 # EOM(1) + .byte 0 # EOM(2) + .byte 11 # Abbreviation Code + .byte 5 # DW_TAG_formal_parameter + .byte 0 # DW_CHILDREN_no + .byte 3 # DW_AT_name + .byte 37 # DW_FORM_strx1 + .byte 58 # DW_AT_decl_file + .byte 11 # DW_FORM_data1 + .byte 59 # DW_AT_decl_line + .byte 11 # DW_FORM_data1 + .byte 73 # DW_AT_type + .byte 19 # DW_FORM_ref4 + .byte 0 # EOM(1) + .byte 0 # EOM(2) + .byte 12 # Abbreviation Code + .byte 46 # DW_TAG_subprogram + .byte 1 # DW_CHILDREN_yes + .byte 17 # DW_AT_low_pc + .byte 27 # DW_FORM_addrx + .byte 18 # DW_AT_high_pc + .byte 6 # DW_FORM_data4 + .byte 64 # DW_AT_frame_base + .byte 24 # DW_FORM_exprloc + .byte 49 # DW_AT_abstract_origin + .byte 19 # DW_FORM_ref4 + .byte 0 # EOM(1) + .byte 0 # EOM(2) + .byte 13 # Abbreviation Code + .byte 5 # DW_TAG_formal_parameter + .byte 0 # DW_CHILDREN_no + .byte 2 # DW_AT_location + .byte 24 # DW_FORM_exprloc + .byte 49 # DW_AT_abstract_origin + .byte 19 # DW_FORM_ref4 + .byte 0 # EOM(1) + .byte 0 # EOM(2) + .byte 0 # EOM(3) + .section .debug_info,"",@progbits +.Lcu_begin0: + .long .Ldebug_info_end0-.Ldebug_info_start0 # Length of Unit +.Ldebug_info_start0: + .short 5 # DWARF version number + .byte 1 # DWARF Unit Type + .byte 8 # Address Size (in bytes) + .long .debug_abbrev # Offset Into Abbrev. Section + .byte 1 # Abbrev [1] 0xc:0x6d DW_TAG_compile_unit + .byte 0 # DW_AT_producer + .short 33 # DW_AT_language + .byte 1 # DW_AT_name + .long .Lstr_offsets_base0 # DW_AT_str_offsets_base + .long .Lline_table_start0 # DW_AT_stmt_list + .byte 2 # DW_AT_comp_dir + .byte 0 # DW_AT_low_pc + .long .Lfunc_end0-.Lfunc_begin0 # DW_AT_high_pc + .long .Laddr_table_base0 # DW_AT_addr_base + .byte 2 # Abbrev [2] 0x23:0x47 DW_TAG_subprogram + .byte 0 # DW_AT_low_pc + .long .Lfunc_end0-.Lfunc_begin0 # DW_AT_high_pc + .byte 1 # DW_AT_frame_base + .byte 86 + .byte 8 # DW_AT_name + .byte 1 # DW_AT_decl_file + .byte 2 # DW_AT_decl_line + .long .debug_info+174 # DW_AT_type + # DW_AT_external + .byte 3 # Abbrev [3] 0x32:0xb DW_TAG_formal_parameter + .byte 2 # DW_AT_location + .byte 145 + .byte 116 + .byte 9 # DW_AT_name + .byte 1 # DW_AT_decl_file + .byte 2 # DW_AT_decl_line + .long .debug_info+174 # DW_AT_type + .byte 4 # Abbrev [4] 0x3d:0xb DW_TAG_formal_parameter + .byte 2 # DW_AT_location + .byte 145 + .byte 104 + .byte 10 # DW_AT_name + .byte 1 # DW_AT_decl_file + .byte 2 # DW_AT_decl_line + .long 106 # DW_AT_type + .byte 5 # Abbrev [5] 0x48:0xb DW_TAG_variable + .byte 2 # DW_AT_location + .byte 145 + .byte 124 + .byte 7 # DW_AT_name + .byte 1 # DW_AT_decl_file + .byte 3 # DW_AT_decl_line + .long .debug_info+174 # DW_AT_type + .byte 6 # Abbrev [6] 0x53:0x16 DW_TAG_inlined_subroutine + .long .debug_info+156 # DW_AT_abstract_origin + .byte 1 # DW_AT_low_pc + .long .Ltmp2-.Ltmp1 # DW_AT_high_pc + .byte 1 # DW_AT_call_file + .byte 4 # DW_AT_call_line + .byte 32 # DW_AT_call_column + .byte 7 # Abbrev [7] 0x60:0x8 DW_TAG_formal_parameter + .byte 2 # DW_AT_location + .byte 145 + .byte 120 + .long .debug_info+165 # DW_AT_abstract_origin + .byte 0 # End Of Children Mark + .byte 0 # End Of Children Mark + .byte 8 # Abbrev [8] 0x6a:0x5 DW_TAG_pointer_type + .long 111 # DW_AT_type + .byte 8 # Abbrev [8] 0x6f:0x5 DW_TAG_pointer_type + .long 116 # DW_AT_type + .byte 9 # Abbrev [9] 0x74:0x4 DW_TAG_base_type + .byte 11 # DW_AT_name + .byte 6 # DW_AT_encoding + .byte 1 # DW_AT_byte_size + .byte 0 # End Of Children Mark +.Ldebug_info_end0: +.Lcu_begin1: + .long .Ldebug_info_end1-.Ldebug_info_start1 # Length of Unit +.Ldebug_info_start1: + .short 5 # DWARF version number + .byte 1 # DWARF Unit Type + .byte 8 # Address Size (in bytes) + .long .debug_abbrev # Offset Into Abbrev. Section + .byte 1 # Abbrev [1] 0xc:0x43 DW_TAG_compile_unit + .byte 0 # DW_AT_producer + .short 33 # DW_AT_language + .byte 3 # DW_AT_name + .long .Lstr_offsets_base0 # DW_AT_str_offsets_base + .long .Lline_table_start0 # DW_AT_stmt_list + .byte 2 # DW_AT_comp_dir + .byte 2 # DW_AT_low_pc + .long .Lfunc_end1-.Lfunc_begin1 # DW_AT_high_pc + .long .Laddr_table_base0 # DW_AT_addr_base + .byte 10 # Abbrev [10] 0x23:0x12 DW_TAG_subprogram + .byte 4 # DW_AT_linkage_name + .byte 5 # DW_AT_name + .byte 2 # DW_AT_decl_file + .byte 1 # DW_AT_decl_line + .long 53 # DW_AT_type + # DW_AT_external + # DW_AT_inline + .byte 11 # Abbrev [11] 0x2c:0x8 DW_TAG_formal_parameter + .byte 7 # DW_AT_name + .byte 2 # DW_AT_decl_file + .byte 1 # DW_AT_decl_line + .long 53 # DW_AT_type + .byte 0 # End Of Children Mark + .byte 9 # Abbrev [9] 0x35:0x4 DW_TAG_base_type + .byte 6 # DW_AT_name + .byte 5 # DW_AT_encoding + .byte 4 # DW_AT_byte_size + .byte 12 # Abbrev [12] 0x39:0x15 DW_TAG_subprogram + .byte 2 # DW_AT_low_pc + .long .Lfunc_end1-.Lfunc_begin1 # DW_AT_high_pc + .byte 1 # DW_AT_frame_base + .byte 86 + .long 35 # DW_AT_abstract_origin + .byte 13 # Abbrev [13] 0x45:0x8 DW_TAG_formal_parameter + .byte 2 # DW_AT_location + .byte 145 + .byte 124 + .long 44 # DW_AT_abstract_origin + .byte 0 # End Of Children Mark + .byte 0 # End Of Children Mark +.Ldebug_info_end1: + .section .debug_str_offsets,"",@progbits + .long 52 # Length of String Offsets Set + .short 5 + .short 0 +.Lstr_offsets_base0: + .section .debug_str,"MS",@progbits,1 +.Linfo_string0: + .asciz "clang version 19.0.0git" # string offset=0 +.Linfo_string1: + .asciz "main.cpp" # string offset=24 +.Linfo_string2: + .asciz "/home/ayermolo/local/tasks/T182867349" # string offset=33 +.Linfo_string3: + .asciz "helper.cpp" # string offset=71 +.Linfo_string4: + .asciz "_Z3fooi" # string offset=82 +.Linfo_string5: + .asciz "foo" # string offset=90 +.Linfo_string6: + .asciz "int" # string offset=94 +.Linfo_string7: + .asciz "i" # string offset=98 +.Linfo_string8: + .asciz "main" # string offset=100 +.Linfo_string9: + .asciz "argc" # string offset=105 +.Linfo_string10: + .asciz "argv" # string offset=110 +.Linfo_string11: + .asciz "char" # string offset=115 + .section .debug_str_offsets,"",@progbits + .long .Linfo_string0 + .long .Linfo_string1 + .long .Linfo_string2 + .long .Linfo_string3 + .long .Linfo_string4 + .long .Linfo_string5 + .long .Linfo_string6 + .long .Linfo_string7 + .long .Linfo_string8 + .long .Linfo_string9 + .long .Linfo_string10 + .long .Linfo_string11 + .section .debug_addr,"",@progbits + .long .Ldebug_addr_end0-.Ldebug_addr_start0 # Length of contribution +.Ldebug_addr_start0: + .short 5 # DWARF version number + .byte 8 # Address size + .byte 0 # Segment selector size +.Laddr_table_base0: + .quad .Lfunc_begin0 + .quad .Ltmp1 + .quad .Lfunc_begin1 +.Ldebug_addr_end0: + .section .debug_names,"",@progbits + .long .Lnames_end0-.Lnames_start0 # Header: unit length +.Lnames_start0: + .short 5 # Header: version + .short 0 # Header: padding + .long 2 # Header: compilation unit count + .long 0 # Header: local type unit count + .long 0 # Header: foreign type unit count + .long 5 # Header: bucket count + .long 5 # Header: name count + .long .Lnames_abbrev_end0-.Lnames_abbrev_start0 # Header: abbreviation table size + .long 8 # Header: augmentation string size + .ascii "LLVM0700" # Header: augmentation string + .long .Lcu_begin0 # Compilation unit 0 + .long .Lcu_begin1 # Compilation unit 1 + .long 0 # Bucket 0 + .long 1 # Bucket 1 + .long 0 # Bucket 2 + .long 3 # Bucket 3 + .long 4 # Bucket 4 + .long 2090499946 # Hash in Bucket 1 + .long -1257882370 # Hash in Bucket 1 + .long 193495088 # Hash in Bucket 3 + .long 193491849 # Hash in Bucket 4 + .long 2090147939 # Hash in Bucket 4 + .long .Linfo_string8 # String in Bucket 1: main + .long .Linfo_string4 # String in Bucket 1: _Z3fooi + .long .Linfo_string6 # String in Bucket 3: int + .long .Linfo_string5 # String in Bucket 4: foo + .long .Linfo_string11 # String in Bucket 4: char + .long .Lnames1-.Lnames_entries0 # Offset in Bucket 1 + .long .Lnames3-.Lnames_entries0 # Offset in Bucket 1 + .long .Lnames0-.Lnames_entries0 # Offset in Bucket 3 + .long .Lnames2-.Lnames_entries0 # Offset in Bucket 4 + .long .Lnames4-.Lnames_entries0 # Offset in Bucket 4 +.Lnames_abbrev_start0: + .byte 1 # Abbrev code + .byte 46 # DW_TAG_subprogram + .byte 1 # DW_IDX_compile_unit + .byte 11 # DW_FORM_data1 + .byte 3 # DW_IDX_die_offset + .byte 19 # DW_FORM_ref4 + .byte 4 # DW_IDX_parent + .byte 25 # DW_FORM_flag_present + .byte 0 # End of abbrev + .byte 0 # End of abbrev + .byte 2 # Abbrev code + .byte 29 # DW_TAG_inlined_subroutine + .byte 1 # DW_IDX_compile_unit + .byte 11 # DW_FORM_data1 + .byte 3 # DW_IDX_die_offset + .byte 19 # DW_FORM_ref4 + .byte 4 # DW_IDX_parent + .byte 19 # DW_FORM_ref4 + .byte 0 # End of abbrev + .byte 0 # End of abbrev + .byte 3 # Abbrev code + .byte 36 # DW_TAG_base_type + .byte 1 # DW_IDX_compile_unit + .byte 11 # DW_FORM_data1 + .byte 3 # DW_IDX_die_offset + .byte 19 # DW_FORM_ref4 + .byte 4 # DW_IDX_parent + .byte 25 # DW_FORM_flag_present + .byte 0 # End of abbrev + .byte 0 # End of abbrev + .byte 0 # End of abbrev list +.Lnames_abbrev_end0: +.Lnames_entries0: +.Lnames1: +.L3: + .byte 1 # Abbreviation code + .byte 0 # DW_IDX_compile_unit + .long 35 # DW_IDX_die_offset + .byte 0 # DW_IDX_parent + # End of list: main +.Lnames3: +.L0: + .byte 1 # Abbreviation code + .byte 1 # DW_IDX_compile_unit + .long 57 # DW_IDX_die_offset +.L2: # DW_IDX_parent + .byte 2 # Abbreviation code + .byte 0 # DW_IDX_compile_unit + .long 83 # DW_IDX_die_offset + .long .L3-.Lnames_entries0 # DW_IDX_parent + .byte 0 # End of list: _Z3fooi +.Lnames0: +.L4: + .byte 3 # Abbreviation code + .byte 1 # DW_IDX_compile_unit + .long 53 # DW_IDX_die_offset + .byte 0 # DW_IDX_parent + # End of list: int +.Lnames2: + .byte 1 # Abbreviation code + .byte 1 # DW_IDX_compile_unit + .long 57 # DW_IDX_die_offset + .byte 2 # DW_IDX_parent + # Abbreviation code + .byte 0 # DW_IDX_compile_unit + .long 83 # DW_IDX_die_offset + .long .L3-.Lnames_entries0 # DW_IDX_parent + .byte 0 # End of list: foo +.Lnames4: +.L1: + .byte 3 # Abbreviation code + .byte 0 # DW_IDX_compile_unit + .long 116 # DW_IDX_die_offset + .byte 0 # DW_IDX_parent + # End of list: char + .p2align 2, 0x0 +.Lnames_end0: + .ident "clang version 19.0.0git" + .ident "clang version 19.0.0git" + .section ".note.GNU-stack","",@progbits + .section .debug_line,"",@progbits +.Lline_table_start0: -- GitLab From dcbddc25250158469c5635ad2ae4095faef53dfd Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Fri, 22 Mar 2024 16:13:58 -0500 Subject: [PATCH 105/306] [Libomptarget] Unify and simplify plugin CMake (#86191) Summary: This patch reworks the CMake handling for building plugins. All this does is pull a lot of shared and common logic into a single helper function. This also simplifies the OMPT libraries from being built separately instead of just added. --- .../plugins-nextgen/CMakeLists.txt | 59 ++++++++++++++ .../plugins-nextgen/amdgpu/CMakeLists.txt | 80 +++---------------- .../plugins-nextgen/common/CMakeLists.txt | 54 ++----------- .../common/OMPT/CMakeLists.txt | 70 ---------------- .../plugins-nextgen/cuda/CMakeLists.txt | 40 ++-------- .../plugins-nextgen/host/CMakeLists.txt | 29 +------ 6 files changed, 87 insertions(+), 245 deletions(-) delete mode 100644 openmp/libomptarget/plugins-nextgen/common/OMPT/CMakeLists.txt diff --git a/openmp/libomptarget/plugins-nextgen/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/CMakeLists.txt index 75540f055844..8e8b040fd562 100644 --- a/openmp/libomptarget/plugins-nextgen/CMakeLists.txt +++ b/openmp/libomptarget/plugins-nextgen/CMakeLists.txt @@ -10,7 +10,66 @@ # ##===----------------------------------------------------------------------===## +# Common interface to handle creating a plugin library. +set(common_dir ${CMAKE_CURRENT_SOURCE_DIR}/common) add_subdirectory(common) +function(add_target_library target_name lib_name) + llvm_map_components_to_libnames(llvm_libs + ${LLVM_TARGETS_TO_BUILD} + AggressiveInstCombine + Analysis + BinaryFormat + BitReader + BitWriter + CodeGen + Core + Extensions + InstCombine + Instrumentation + IPO + IRReader + Linker + MC + Object + Passes + Remarks + ScalarOpts + Support + Target + TargetParser + TransformUtils + Vectorize + ) + + add_llvm_library(${target_name} SHARED + NO_INSTALL_RPATH + BUILDTREE_ONLY + ) + + llvm_update_compile_flags(${target_name}) + target_link_libraries(${target_name} PUBLIC + PluginCommon ${llvm_libs} ${OPENMP_PTHREAD_LIB}) + + target_compile_definitions(${target_name} PRIVATE TARGET_NAME=${lib_name}) + target_compile_definitions(${target_name} PRIVATE + DEBUG_PREFIX="TARGET ${lib_name} RTL") + + if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") + # On FreeBSD, the 'environ' symbol is undefined at link time, but resolved by + # the dynamic linker at runtime. Therefore, allow the symbol to be undefined + # when creating a shared library. + target_link_libraries(${target_name} PRIVATE "-Wl,--allow-shlib-undefined") + else() + target_link_libraries(${target_name} PRIVATE "-Wl,-z,defs") + endif() + + if(LIBOMP_HAVE_VERSION_SCRIPT_FLAG) + target_link_libraries(${target_name} PRIVATE + "-Wl,--version-script=${common_dir}/../exports") + endif() + set_target_properties(${target_name} PROPERTIES CXX_VISIBILITY_PRESET protected) +endfunction() + add_subdirectory(amdgpu) add_subdirectory(cuda) add_subdirectory(host) diff --git a/openmp/libomptarget/plugins-nextgen/amdgpu/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/amdgpu/CMakeLists.txt index 8fbfe4d9b13f..40df77102c78 100644 --- a/openmp/libomptarget/plugins-nextgen/amdgpu/CMakeLists.txt +++ b/openmp/libomptarget/plugins-nextgen/amdgpu/CMakeLists.txt @@ -27,76 +27,23 @@ if(NOT (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86_64)|(ppc64le)|(aarch64)$" AND CMAKE return() endif() -################################################################################ -# Define the suffix for the runtime messaging dumps. -add_definitions(-DTARGET_NAME=AMDGPU) - -# Define debug prefix. TODO: This should be automatized in the Debug.h but it -# requires changing the original plugins. -add_definitions(-DDEBUG_PREFIX="TARGET AMDGPU RTL") +# Create the library and add the default arguments. +add_target_library(omptarget.rtl.amdgpu AMDGPU) -set(LIBOMPTARGET_DLOPEN_LIBHSA OFF) -option(LIBOMPTARGET_FORCE_DLOPEN_LIBHSA "Build with dlopened libhsa" ${LIBOMPTARGET_DLOPEN_LIBHSA}) - -if (${hsa-runtime64_FOUND} AND NOT LIBOMPTARGET_FORCE_DLOPEN_LIBHSA) - libomptarget_say("Building AMDGPU NextGen plugin linked against libhsa") - set(LIBOMPTARGET_EXTRA_SOURCE) - set(LIBOMPTARGET_DEP_LIBRARIES hsa-runtime64::hsa-runtime64) -else() - libomptarget_say("Building AMDGPU NextGen plugin for dlopened libhsa") - include_directories(dynamic_hsa) - set(LIBOMPTARGET_EXTRA_SOURCE dynamic_hsa/hsa.cpp) - set(LIBOMPTARGET_DEP_LIBRARIES) -endif() +target_sources(omptarget.rtl.amdgpu PRIVATE src/rtl.cpp) +target_include_directories(omptarget.rtl.amdgpu PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/utils) -if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") - # On FreeBSD, the 'environ' symbol is undefined at link time, but resolved by - # the dynamic linker at runtime. Therefore, allow the symbol to be undefined - # when creating a shared library. - set(LDFLAGS_UNDEFINED "-Wl,--allow-shlib-undefined") +option(LIBOMPTARGET_FORCE_DLOPEN_LIBHSA "Build with dlopened libhsa" OFF) +if(hsa-runtime64_FOUND AND NOT LIBOMPTARGET_FORCE_DLOPEN_LIBHSA) + libomptarget_say("Building AMDGPU plugin linked against libhsa") + target_link_libraries(omptarget.rtl.amdgpu PRIVATE hsa-runtime64::hsa-runtime64) else() - set(LDFLAGS_UNDEFINED "-Wl,-z,defs") + libomptarget_say("Building AMDGPU plugin for dlopened libhsa") + target_include_directories(omptarget.rtl.amdgpu PRIVATE dynamic_hsa) + target_sources(omptarget.rtl.amdgpu PRIVATE dynamic_hsa/hsa.cpp) endif() -add_llvm_library(omptarget.rtl.amdgpu SHARED - src/rtl.cpp - ${LIBOMPTARGET_EXTRA_SOURCE} - - ADDITIONAL_HEADER_DIRS - ${LIBOMPTARGET_INCLUDE_DIR} - ${CMAKE_CURRENT_SOURCE_DIR}/utils - - LINK_COMPONENTS - Support - Object - - LINK_LIBS - PRIVATE - PluginCommon - ${LIBOMPTARGET_DEP_LIBRARIES} - ${OPENMP_PTHREAD_LIB} - ${LDFLAGS_UNDEFINED} - - NO_INSTALL_RPATH - BUILDTREE_ONLY -) - -if ((OMPT_TARGET_DEFAULT) AND (LIBOMPTARGET_OMPT_SUPPORT)) - target_link_libraries(omptarget.rtl.amdgpu PRIVATE OMPT) -endif() - -if (LIBOMP_HAVE_VERSION_SCRIPT_FLAG) - target_link_libraries(omptarget.rtl.amdgpu PRIVATE - "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/../exports") -endif() - -target_include_directories( - omptarget.rtl.amdgpu - PRIVATE - ${LIBOMPTARGET_INCLUDE_DIR} - ${CMAKE_CURRENT_SOURCE_DIR}/utils -) - # Configure testing for the AMDGPU plugin. We will build tests if we could a # functional AMD GPU on the system, or if manually specifies by the user. option(LIBOMPTARGET_FORCE_AMDGPU_TESTS "Build AMDGPU libomptarget tests" OFF) @@ -114,5 +61,4 @@ endif() # Install plugin under the lib destination folder. install(TARGETS omptarget.rtl.amdgpu LIBRARY DESTINATION "${OPENMP_INSTALL_LIBDIR}") set_target_properties(omptarget.rtl.amdgpu PROPERTIES - INSTALL_RPATH "$ORIGIN" BUILD_RPATH "$ORIGIN:${CMAKE_CURRENT_BINARY_DIR}/.." - CXX_VISIBILITY_PRESET protected) + INSTALL_RPATH "$ORIGIN" BUILD_RPATH "$ORIGIN:${CMAKE_CURRENT_BINARY_DIR}/..") diff --git a/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt index 085d44307165..0420d0e6f1f8 100644 --- a/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt +++ b/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt @@ -26,45 +26,6 @@ foreach(Target ${TargetsSupported}) target_compile_definitions(PluginCommon PRIVATE "LIBOMPTARGET_JIT_${Target}") endforeach() -# This is required when using LLVM libraries. -llvm_update_compile_flags(PluginCommon) - -if (LLVM_LINK_LLVM_DYLIB) - set(llvm_libs LLVM) -else() - llvm_map_components_to_libnames(llvm_libs - ${LLVM_TARGETS_TO_BUILD} - AggressiveInstCombine - Analysis - BinaryFormat - BitReader - BitWriter - CodeGen - Core - Extensions - InstCombine - Instrumentation - IPO - IRReader - Linker - MC - Object - Passes - Remarks - ScalarOpts - Support - Target - TargetParser - TransformUtils - Vectorize - ) -endif() - -target_link_libraries(PluginCommon - PUBLIC - ${llvm_libs} -) - # Include the RPC server from the `libc` project if availible. if(TARGET llvmlibc_rpc_server AND ${LIBOMPTARGET_GPU_LIBC_SUPPORT}) target_link_libraries(PluginCommon PRIVATE llvmlibc_rpc_server) @@ -82,8 +43,10 @@ elseif(${LIBOMPTARGET_GPU_LIBC_SUPPORT}) endif() endif() -if ((OMPT_TARGET_DEFAULT) AND (LIBOMPTARGET_OMPT_SUPPORT)) - target_link_libraries(PluginCommon PUBLIC OMPT) +# If we have OMPT enabled include it in the list of sources. +if (OMPT_TARGET_DEFAULT AND LIBOMPTARGET_OMPT_SUPPORT) + target_sources(PluginCommon PRIVATE OMPT/OmptCallback.cpp) + target_include_directories(PluginCommon PRIVATE OMPT) endif() # Define the TARGET_NAME and DEBUG_PREFIX. @@ -95,16 +58,11 @@ target_compile_definitions(PluginCommon PRIVATE target_compile_options(PluginCommon PUBLIC ${offload_compile_flags}) target_link_options(PluginCommon PUBLIC ${offload_link_flags}) -target_include_directories(PluginCommon - PRIVATE - ${LIBOMPTARGET_INCLUDE_DIR} - PUBLIC +target_include_directories(PluginCommon PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include + ${LIBOMPTARGET_INCLUDE_DIR} ) set_target_properties(PluginCommon PROPERTIES POSITION_INDEPENDENT_CODE ON CXX_VISIBILITY_PRESET protected) - -add_subdirectory(OMPT) - diff --git a/openmp/libomptarget/plugins-nextgen/common/OMPT/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/common/OMPT/CMakeLists.txt deleted file mode 100644 index be4c743665b3..000000000000 --- a/openmp/libomptarget/plugins-nextgen/common/OMPT/CMakeLists.txt +++ /dev/null @@ -1,70 +0,0 @@ -##===----------------------------------------------------------------------===## -# -# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -# -##===----------------------------------------------------------------------===## -# -# Aggregation of parts which can be used by OpenMP tools -# -##===----------------------------------------------------------------------===## - -# NOTE: Don't try to build `OMPT` using `add_llvm_library` because we -# don't want to export `OMPT` while `add_llvm_library` requires that. -add_library(OMPT OBJECT - OmptCallback.cpp) - -# This is required when using LLVM libraries. -llvm_update_compile_flags(OMPT) - -if (LLVM_LINK_LLVM_DYLIB) - set(llvm_libs LLVM) -else() - llvm_map_components_to_libnames(llvm_libs - ${LLVM_TARGETS_TO_BUILD} - AggressiveInstCombine - Analysis - BinaryFormat - BitReader - BitWriter - CodeGen - Core - Extensions - InstCombine - Instrumentation - IPO - IRReader - Linker - MC - Object - Passes - Remarks - ScalarOpts - Support - Target - TargetParser - TransformUtils - Vectorize - ) -endif() - -target_link_libraries(OMPT - PUBLIC - ${llvm_libs} -) - -# Define the TARGET_NAME and DEBUG_PREFIX. -target_compile_definitions(OMPT PRIVATE - TARGET_NAME="OMPT" - DEBUG_PREFIX="OMPT" -) - -target_include_directories(OMPT - INTERFACE ${CMAKE_CURRENT_SOURCE_DIR} - PRIVATE ${LIBOMPTARGET_INCLUDE_DIR} -) - -set_target_properties(OMPT PROPERTIES - POSITION_INDEPENDENT_CODE ON - CXX_VISIBILITY_PRESET protected) diff --git a/openmp/libomptarget/plugins-nextgen/cuda/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/cuda/CMakeLists.txt index 2bfb47168a7f..b3530462aa19 100644 --- a/openmp/libomptarget/plugins-nextgen/cuda/CMakeLists.txt +++ b/openmp/libomptarget/plugins-nextgen/cuda/CMakeLists.txt @@ -23,34 +23,12 @@ endif() libomptarget_say("Building CUDA NextGen offloading plugin.") -set(LIBOMPTARGET_DLOPEN_LIBCUDA OFF) -option(LIBOMPTARGET_FORCE_DLOPEN_LIBCUDA "Build with dlopened libcuda" ${LIBOMPTARGET_DLOPEN_LIBCUDA}) - -add_llvm_library(omptarget.rtl.cuda SHARED - src/rtl.cpp - - LINK_COMPONENTS - Support - Object - - LINK_LIBS PRIVATE - PluginCommon - ${OPENMP_PTHREAD_LIB} - - NO_INSTALL_RPATH - BUILDTREE_ONLY -) - -if ((OMPT_TARGET_DEFAULT) AND (LIBOMPTARGET_OMPT_SUPPORT)) - target_link_libraries(omptarget.rtl.cuda PRIVATE OMPT) -endif() - -if (LIBOMP_HAVE_VERSION_SCRIPT_FLAG) - target_link_libraries(omptarget.rtl.cuda PRIVATE - "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/../exports,-z,defs") -endif() +# Create the library and add the default arguments. +add_target_library(omptarget.rtl.cuda CUDA) +target_sources(omptarget.rtl.cuda PRIVATE src/rtl.cpp) +option(LIBOMPTARGET_FORCE_DLOPEN_LIBCUDA "Build with dlopened libcuda" OFF) if(LIBOMPTARGET_DEP_CUDA_FOUND AND NOT LIBOMPTARGET_FORCE_DLOPEN_LIBCUDA) libomptarget_say("Building CUDA plugin linked against libcuda") target_link_libraries(omptarget.rtl.cuda PRIVATE CUDA::cuda_driver) @@ -60,13 +38,6 @@ else() target_sources(omptarget.rtl.cuda PRIVATE dynamic_cuda/cuda.cpp) endif() -# Define debug prefix. TODO: This should be automatized in the Debug.h but it -# requires changing the original plugins. -target_compile_definitions(omptarget.rtl.cuda PRIVATE TARGET_NAME="CUDA") -target_compile_definitions(omptarget.rtl.cuda PRIVATE DEBUG_PREFIX="TARGET CUDA RTL") - -target_include_directories(omptarget.rtl.cuda PRIVATE ${LIBOMPTARGET_INCLUDE_DIR}) - # Configure testing for the CUDA plugin. We will build tests if we could a # functional NVIDIA GPU on the system, or if manually specifies by the user. option(LIBOMPTARGET_FORCE_NVIDIA_TESTS "Build NVIDIA libomptarget tests" OFF) @@ -84,5 +55,4 @@ endif() # Install plugin under the lib destination folder. install(TARGETS omptarget.rtl.cuda LIBRARY DESTINATION "${OPENMP_INSTALL_LIBDIR}") set_target_properties(omptarget.rtl.cuda PROPERTIES - INSTALL_RPATH "$ORIGIN" BUILD_RPATH "$ORIGIN:${CMAKE_CURRENT_BINARY_DIR}/.." - CXX_VISIBILITY_PRESET protected) + INSTALL_RPATH "$ORIGIN" BUILD_RPATH "$ORIGIN:${CMAKE_CURRENT_BINARY_DIR}/..") diff --git a/openmp/libomptarget/plugins-nextgen/host/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/host/CMakeLists.txt index 58a79898ff80..d30680e10431 100644 --- a/openmp/libomptarget/plugins-nextgen/host/CMakeLists.txt +++ b/openmp/libomptarget/plugins-nextgen/host/CMakeLists.txt @@ -2,11 +2,6 @@ if(NOT CMAKE_SYSTEM_NAME MATCHES "Linux") return() endif() - # build_generic_elf64("s390x" "S390X" "s390x" "systemz" "s390x-ibm-linux-gnu" "22") - # build_generic_elf64("aarch64" "aarch64" "aarch64" "aarch64" "aarch64-unknown-linux-gnu" "183") - # build_generic_elf64("ppc64" "PPC64" "ppc64" "ppc64" "powerpc64-ibm-linux-gnu" "21") - # build_generic_elf64("x86_64" "x86_64" "x86_64" "x86_64" "x86_64-pc-linux-gnu" "62") - # build_generic_elf64("ppc64le" "PPC64le" "ppc64" "ppc64le" "powerpc64le-ibm-linux-gnu" "21") set(supported_targets x86_64 aarch64 ppc64 ppc64le s390x) if(NOT ${CMAKE_SYSTEM_PROCESSOR} IN_LIST supported_targets) libomptarget_say("Not building ${machine} NextGen offloading plugin") @@ -18,16 +13,10 @@ if(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64le$") set(machine ppc64) endif() -add_llvm_library(omptarget.rtl.${machine} SHARED - src/rtl.cpp - ADDITIONAL_HEADER_DIRS - ${LIBOMPTARGET_INCLUDE_DIR} - LINK_LIBS PRIVATE - PluginCommon - ${OPENMP_PTHREAD_LIB} - NO_INSTALL_RPATH - BUILDTREE_ONLY -) +# Create the library and add the default arguments. +add_target_library(omptarget.rtl.${machine} ${machine}) + +target_sources(omptarget.rtl.${machine} PRIVATE src/rtl.cpp) if(LIBOMPTARGET_DEP_LIBFFI_FOUND) libomptarget_say("Building ${machine} plugin linked with libffi") @@ -42,10 +31,6 @@ else() target_include_directories(omptarget.rtl.${machine} PRIVATE dynamic_ffi) endif() -if(OMPT_TARGET_DEFAULT AND LIBOMPTARGET_OMPT_SUPPORT) - target_link_libraries(omptarget.rtl.${machine} PRIVATE OMPT) -endif() - if(LIBOMP_HAVE_VERSION_SCRIPT_FLAG) target_link_libraries(omptarget.rtl.${machine} PRIVATE "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/../exports") @@ -70,12 +55,6 @@ else() libomptarget_say("Not generating ${tmachine_name} tests. LibFFI not found.") endif() -# Define macro to be used as prefix of the runtime messages for this target. -target_compile_definitions(omptarget.rtl.${machine} PRIVATE TARGET_NAME=${machine}) -# TODO: This should be automatized in Debug.h. -target_compile_definitions(omptarget.rtl.${machine} PRIVATE - DEBUG_PREFIX="TARGET ${machine} RTL") - # Define the target specific triples and ELF machine values. if(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64le$" OR CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64$") -- GitLab From 215f105ca5d0b42d00bbbc315605b222d63be63a Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Fri, 22 Mar 2024 14:14:43 -0700 Subject: [PATCH 106/306] [MTE] Fix test (#85875) llc runs the stack tagging instrumentation, so if we run opt before, we double instrument --- llvm/test/CodeGen/AArch64/stack-tagging-stack-coloring.ll | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/llvm/test/CodeGen/AArch64/stack-tagging-stack-coloring.ll b/llvm/test/CodeGen/AArch64/stack-tagging-stack-coloring.ll index 81349620fb77..5d1c91e434dc 100644 --- a/llvm/test/CodeGen/AArch64/stack-tagging-stack-coloring.ll +++ b/llvm/test/CodeGen/AArch64/stack-tagging-stack-coloring.ll @@ -1,17 +1,15 @@ ; Test that storage for allocas with disjoint lifetimes is reused with stack ; tagging. -; RUN: opt -S -aarch64-stack-tagging -stack-tagging-use-stack-safety=0 %s -o - | \ -; RUN: llc --mattr=+mte -no-stack-coloring=false -o - | \ +; RUN: llc --mattr=+mte -no-stack-coloring=false -stack-tagging-use-stack-safety=0 -o - %s | \ ; RUN: FileCheck %s --check-prefix=COLOR -; RUN: opt -S -aarch64-stack-tagging %s -stack-tagging-use-stack-safety=0 -o - | \ -; RUN: llc --mattr=+mte -no-stack-coloring=true -o - | \ +; RUN: llc --mattr=+mte -no-stack-coloring=true -stack-tagging-use-stack-safety=0 -o - %s | \ ; RUN: FileCheck %s --check-prefix=NOCOLOR target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" target triple = "aarch64" -; COLOR: sub sp, sp, #208 +; COLOR: sub sp, sp, #192 ; NOCOLOR: sub sp, sp, #336 define i32 @myCall_w2(i32 %in) sanitize_memtag { -- GitLab From 14be4930c10bcc6f6c0096097350cb3cafff9661 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Fri, 22 Mar 2024 14:17:51 -0700 Subject: [PATCH 107/306] [bazel] Make compiler-rt analyze on macOS (#86001) Previously the select above would fail for non-linux platforms if you did a `bazel build @llvm-project//...`, now this target specifies that it's only supported on the linux platform through bazel's `target_compatible_with` feature. This makes all targets in the tree be ignored when building on incompatible platforms (and fail if built directly) --- utils/bazel/llvm-project-overlay/compiler-rt/BUILD.bazel | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/utils/bazel/llvm-project-overlay/compiler-rt/BUILD.bazel b/utils/bazel/llvm-project-overlay/compiler-rt/BUILD.bazel index 9bdd454e1e36..577e6c033b4e 100644 --- a/utils/bazel/llvm-project-overlay/compiler-rt/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/compiler-rt/BUILD.bazel @@ -19,6 +19,10 @@ cc_library( ], # Will raise error unless supported platforms. }), + target_compatible_with = select({ + "@platforms//os:linux": [], + "//conditions:default": ["@platforms//:incompatible"], + }), ) WIN32_ONLY_FILES = [ -- GitLab From 85af772f3b4067fce703b33cee0e2cdafc74a6d6 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Fri, 22 Mar 2024 16:38:10 -0500 Subject: [PATCH 108/306] [Libomptarget][FIX] Fix unintentinally used PUBLIC interface Summary: This was supposed to be private and caused some issues with certain configs. --- openmp/libomptarget/plugins-nextgen/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openmp/libomptarget/plugins-nextgen/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/CMakeLists.txt index 8e8b040fd562..c19fd80592d6 100644 --- a/openmp/libomptarget/plugins-nextgen/CMakeLists.txt +++ b/openmp/libomptarget/plugins-nextgen/CMakeLists.txt @@ -47,7 +47,7 @@ function(add_target_library target_name lib_name) ) llvm_update_compile_flags(${target_name}) - target_link_libraries(${target_name} PUBLIC + target_link_libraries(${target_name} PRIVATE PluginCommon ${llvm_libs} ${OPENMP_PTHREAD_LIB}) target_compile_definitions(${target_name} PRIVATE TARGET_NAME=${lib_name}) -- GitLab From 5d0d9eb52dbb3bcf6f500c7b18d58c8bdf6659ce Mon Sep 17 00:00:00 2001 From: Ellis Hoag Date: Fri, 22 Mar 2024 14:47:31 -0700 Subject: [PATCH 109/306] [NFC][BP] Remove unused parameter from function (#86333) Remove the unused parameter `RecDepth` from `runIterations()`. --- llvm/include/llvm/Support/BalancedPartitioning.h | 5 ++--- llvm/lib/Support/BalancedPartitioning.cpp | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/llvm/include/llvm/Support/BalancedPartitioning.h b/llvm/include/llvm/Support/BalancedPartitioning.h index 9738e742f7f1..539d157343fb 100644 --- a/llvm/include/llvm/Support/BalancedPartitioning.h +++ b/llvm/include/llvm/Support/BalancedPartitioning.h @@ -142,9 +142,8 @@ private: std::optional &TP) const; /// Run bisection iterations - void runIterations(const FunctionNodeRange Nodes, unsigned RecDepth, - unsigned LeftBucket, unsigned RightBucket, - std::mt19937 &RNG) const; + void runIterations(const FunctionNodeRange Nodes, unsigned LeftBucket, + unsigned RightBucket, std::mt19937 &RNG) const; /// Run a bisection iteration to improve the optimization goal /// \returns the total number of moved FunctionNodes diff --git a/llvm/lib/Support/BalancedPartitioning.cpp b/llvm/lib/Support/BalancedPartitioning.cpp index f4254b50d26c..141f0034a23f 100644 --- a/llvm/lib/Support/BalancedPartitioning.cpp +++ b/llvm/lib/Support/BalancedPartitioning.cpp @@ -136,7 +136,7 @@ void BalancedPartitioning::bisect(const FunctionNodeRange Nodes, // Split into two and assign to the left and right buckets split(Nodes, LeftBucket); - runIterations(Nodes, RecDepth, LeftBucket, RightBucket, RNG); + runIterations(Nodes, LeftBucket, RightBucket, RNG); // Split nodes wrt the resulting buckets auto NodesMid = @@ -163,7 +163,7 @@ void BalancedPartitioning::bisect(const FunctionNodeRange Nodes, } void BalancedPartitioning::runIterations(const FunctionNodeRange Nodes, - unsigned RecDepth, unsigned LeftBucket, + unsigned LeftBucket, unsigned RightBucket, std::mt19937 &RNG) const { unsigned NumNodes = std::distance(Nodes.begin(), Nodes.end()); -- GitLab From 4652ec0e291ca4ba4ddef3fd59b202646e9a6694 Mon Sep 17 00:00:00 2001 From: Patrick O'Neill <102189596+patrick-rivos@users.noreply.github.com> Date: Fri, 22 Mar 2024 14:52:27 -0700 Subject: [PATCH 110/306] [SLP] Delete vectorized users when tree contains an invalid cost (#86344) --- .../Transforms/Vectorize/SLPVectorizer.cpp | 2 +- .../RISCV/partial-vec-invalid-cost.ll | 57 +++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 llvm/test/Transforms/SLPVectorizer/RISCV/partial-vec-invalid-cost.ll diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 0f7afa2fc25c..f98d15c285a6 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -15959,7 +15959,7 @@ public: LLVM_DEBUG(dbgs() << "SLP: Found cost = " << Cost << " for reduction\n"); if (!Cost.isValid()) - return nullptr; + break; if (Cost >= -SLPCostThreshold) { V.getORE()->emit([&]() { return OptimizationRemarkMissed( diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/partial-vec-invalid-cost.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/partial-vec-invalid-cost.ll new file mode 100644 index 000000000000..31f16801b7a6 --- /dev/null +++ b/llvm/test/Transforms/SLPVectorizer/RISCV/partial-vec-invalid-cost.ll @@ -0,0 +1,57 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt < %s -passes=slp-vectorizer -S | FileCheck %s + +target triple = "riscv64-unknown-linux-gnu" + +define void @partial_vec_invalid_cost() #0 { +; CHECK-LABEL: define void @partial_vec_invalid_cost( +; CHECK-SAME: ) #[[ATTR0:[0-9]+]] { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[LSHR_1:%.*]] = lshr i96 0, 0 +; CHECK-NEXT: [[LSHR_2:%.*]] = lshr i96 0, 0 +; CHECK-NEXT: [[TRUNC_I96_1:%.*]] = trunc i96 [[LSHR_1]] to i32 +; CHECK-NEXT: [[TRUNC_I96_2:%.*]] = trunc i96 [[LSHR_2]] to i32 +; CHECK-NEXT: [[TRUNC_I96_3:%.*]] = trunc i96 0 to i32 +; CHECK-NEXT: [[TRUNC_I96_4:%.*]] = trunc i96 0 to i32 +; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> zeroinitializer) +; CHECK-NEXT: [[OP_RDX:%.*]] = or i32 [[TMP0]], [[TRUNC_I96_2]] +; CHECK-NEXT: [[OP_RDX1:%.*]] = or i32 [[TRUNC_I96_1]], [[TRUNC_I96_3]] +; CHECK-NEXT: [[OP_RDX2:%.*]] = or i32 [[OP_RDX]], [[OP_RDX1]] +; CHECK-NEXT: [[OP_RDX3:%.*]] = or i32 [[OP_RDX2]], [[TRUNC_I96_4]] +; CHECK-NEXT: [[STORE_THIS:%.*]] = zext i32 [[OP_RDX3]] to i96 +; CHECK-NEXT: store i96 [[STORE_THIS]], ptr null, align 16 +; CHECK-NEXT: ret void +; +entry: + + %lshr.1 = lshr i96 0, 0 ; These ops + %lshr.2 = lshr i96 0, 0 ; return an + %add.0 = add i96 0, 0 ; invalid + %add.1 = add i96 0, 0 ; vector cost. + + %trunc.i96.1 = trunc i96 %lshr.1 to i32 ; These ops + %trunc.i96.2 = trunc i96 %lshr.2 to i32 ; return an + %trunc.i96.3 = trunc i96 %add.0 to i32 ; invalid + %trunc.i96.4 = trunc i96 %add.1 to i32 ; vector cost. + + %or.0 = or i32 %trunc.i96.1, %trunc.i96.2 + %or.1 = or i32 %or.0, %trunc.i96.3 + %or.2 = or i32 %or.1, %trunc.i96.4 + + %zext.0 = zext i1 0 to i32 ; These + %zext.1 = zext i1 0 to i32 ; ops + %zext.2 = zext i1 0 to i32 ; are + %zext.3 = zext i1 0 to i32 ; vectorized + + %or.3 = or i32 %or.2, %zext.0 ; users + %or.4 = or i32 %or.3, %zext.1 ; of + %or.5 = or i32 %or.4, %zext.2 ; vectorized + %or.6 = or i32 %or.5, %zext.3 ; ops + + %store.this = zext i32 %or.6 to i96 + + store i96 %store.this, ptr null, align 16 + ret void +} + +attributes #0 = { "target-features"="+v" } -- GitLab From 913e29966bac5fec08998a1acc3e793f9b7bcc12 Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Fri, 22 Mar 2024 15:11:26 -0700 Subject: [PATCH 111/306] [NFC][tsan] Use the result of placement new (#86341) --- compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cpp b/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cpp index 77488f843285..06c34a2e4383 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cpp @@ -200,9 +200,8 @@ void ThreadStart(ThreadState *thr, Tid tid, tid_t os_id, } void ThreadContext::OnStarted(void *arg) { - thr = static_cast(arg); DPrintf("#%d: ThreadStart\n", tid); - new (thr) ThreadState(tid); + thr = new (arg) ThreadState(tid); if (common_flags()->detect_deadlocks) thr->dd_lt = ctx->dd->CreateLogicalThread(tid); thr->tctx = this; -- GitLab From 0ba678a53d3ef7d125f38720a59875035739dc9b Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Fri, 22 Mar 2024 15:12:09 -0700 Subject: [PATCH 112/306] [tsan] Set `thr->is_inited` after SlotAttachAndLock (#86342) Almost NFC. This is symmetrical to `ThreadFinish`, which resets the slot after `is_inited`. --- compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cpp b/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cpp index 06c34a2e4383..5316a7862e44 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cpp @@ -160,6 +160,10 @@ void ThreadStart(ThreadState *thr, Tid tid, tid_t os_id, } Free(thr->tctx->sync); +#if !SANITIZER_GO + thr->is_inited = true; +#endif + uptr stk_addr = 0; uptr stk_size = 0; uptr tls_addr = 0; @@ -205,9 +209,6 @@ void ThreadContext::OnStarted(void *arg) { if (common_flags()->detect_deadlocks) thr->dd_lt = ctx->dd->CreateLogicalThread(tid); thr->tctx = this; -#if !SANITIZER_GO - thr->is_inited = true; -#endif } void ThreadFinish(ThreadState *thr) { -- GitLab From 362d26366d0175f01ffb6085eb747a6e40f01147 Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Fri, 22 Mar 2024 15:24:44 -0700 Subject: [PATCH 113/306] [tsan] Process SIGPROF as sync signal only if thread is alive (#86343) Otherwise it may crash too early. This is followup to #85188 --- compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp b/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp index 2bebe651b994..810ce69663d0 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp @@ -2169,8 +2169,7 @@ static bool is_sync_signal(ThreadSignalContext *sctx, int sig, return false; #endif return sig == SIGSEGV || sig == SIGBUS || sig == SIGILL || sig == SIGTRAP || - sig == SIGABRT || sig == SIGFPE || sig == SIGPIPE || sig == SIGSYS || - sig == SIGPROF; + sig == SIGABRT || sig == SIGFPE || sig == SIGPIPE || sig == SIGSYS; } void sighandler(int sig, __sanitizer_siginfo *info, void *ctx) { @@ -2181,7 +2180,8 @@ void sighandler(int sig, __sanitizer_siginfo *info, void *ctx) { return; } // Don't mess with synchronous signals. - const bool sync = is_sync_signal(sctx, sig, info); + const bool sync = is_sync_signal(sctx, sig, info) || + (sig == SIGPROF && thr->is_inited && !thr->is_dead); if (sync || // If we are in blocking function, we can safely process it now // (but check if we are in a recursive interceptor, -- GitLab From b1e97d60bd5b1d3f994345caa4012ea11c2a0f62 Mon Sep 17 00:00:00 2001 From: Yeoul Na Date: Sat, 23 Mar 2024 07:26:35 +0900 Subject: [PATCH 114/306] Unwrap CountAttributed for debug info (#86017) Fix crash caused by 3eb9ff30959a670559bcba03d149d4c51bf7c9c9 --- clang/lib/CodeGen/CGDebugInfo.cpp | 3 +++ .../test/CodeGen/attr-counted-by-debug-info.c | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 clang/test/CodeGen/attr-counted-by-debug-info.c diff --git a/clang/lib/CodeGen/CGDebugInfo.cpp b/clang/lib/CodeGen/CGDebugInfo.cpp index 07ecaa81c47d..7453ed14aef4 100644 --- a/clang/lib/CodeGen/CGDebugInfo.cpp +++ b/clang/lib/CodeGen/CGDebugInfo.cpp @@ -3463,6 +3463,9 @@ static QualType UnwrapTypeForDebugInfo(QualType T, const ASTContext &C) { case Type::BTFTagAttributed: T = cast(T)->getWrappedType(); break; + case Type::CountAttributed: + T = cast(T)->desugar(); + break; case Type::Elaborated: T = cast(T)->getNamedType(); break; diff --git a/clang/test/CodeGen/attr-counted-by-debug-info.c b/clang/test/CodeGen/attr-counted-by-debug-info.c new file mode 100644 index 000000000000..a6c2b1382b79 --- /dev/null +++ b/clang/test/CodeGen/attr-counted-by-debug-info.c @@ -0,0 +1,18 @@ +// RUN: %clang -emit-llvm -DCOUNTED_BY -S -g %s -o - | FileCheck %s +// RUN: %clang -emit-llvm -S -g %s -o - | FileCheck %s + +#ifdef COUNTED_BY +#define __counted_by(member) __attribute__((__counted_by__(member))) +#else +#define __counted_by(member) +#endif + +struct { + int num_counters; + long value[] __counted_by(num_counters); +} agent_send_response_port_num; + +// CHECK: !DICompositeType(tag: DW_TAG_array_type, baseType: ![[BT:.*]], elements: ![[ELEMENTS:.*]]) +// CHECK: ![[BT]] = !DIBasicType(name: "long", size: {{.*}}, encoding: DW_ATE_signed) +// CHECK: ![[ELEMENTS]] = !{![[COUNT:.*]]} +// CHECK: ![[COUNT]] = !DISubrange(count: -1) \ No newline at end of file -- GitLab From 56197d732e5d5d158fce2f2dfddf3d0bf0d12525 Mon Sep 17 00:00:00 2001 From: Maksim Panchenko Date: Fri, 22 Mar 2024 15:28:54 -0700 Subject: [PATCH 115/306] [BOLT] Skip functions with unsupported Linux kernel features (#86345) Do not overwrite functions with alternative and paravirtual instructions until a proper update support is implemented. --- bolt/lib/Rewrite/LinuxKernelRewriter.cpp | 47 +++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/bolt/lib/Rewrite/LinuxKernelRewriter.cpp b/bolt/lib/Rewrite/LinuxKernelRewriter.cpp index 303e8b18fd32..42df96817275 100644 --- a/bolt/lib/Rewrite/LinuxKernelRewriter.cpp +++ b/bolt/lib/Rewrite/LinuxKernelRewriter.cpp @@ -252,11 +252,17 @@ class LinuxKernelRewriter final : public MetadataRewriter { /// Paravirtual instruction patch sites. Error readParaInstructions(); + Error rewriteParaInstructions(); Error readBugTable(); - /// Read alternative instruction info from .altinstructions. + /// Do no process functions containing instruction annotated with + /// \p Annotation. + void skipFunctionsWithAnnotation(StringRef Annotation) const; + + /// Handle alternative instruction info from .altinstructions. Error readAltInstructions(); + Error rewriteAltInstructions(); /// Read .pci_fixup Error readPCIFixupTable(); @@ -318,6 +324,12 @@ public: if (Error E = rewriteExceptionTable()) return E; + if (Error E = rewriteAltInstructions()) + return E; + + if (Error E = rewriteParaInstructions()) + return E; + if (Error E = rewriteORCTables()) return E; @@ -1126,6 +1138,31 @@ Error LinuxKernelRewriter::readParaInstructions() { return Error::success(); } +void LinuxKernelRewriter::skipFunctionsWithAnnotation( + StringRef Annotation) const { + for (BinaryFunction &BF : llvm::make_second_range(BC.getBinaryFunctions())) { + if (!BC.shouldEmit(BF)) + continue; + for (const BinaryBasicBlock &BB : BF) { + const bool HasAnnotation = llvm::any_of(BB, [&](const MCInst &Inst) { + return BC.MIB->hasAnnotation(Inst, Annotation); + }); + if (HasAnnotation) { + BF.setSimple(false); + break; + } + } + } +} + +Error LinuxKernelRewriter::rewriteParaInstructions() { + // Disable output of functions with paravirtual instructions before the + // rewrite support is complete. + skipFunctionsWithAnnotation("ParaSite"); + + return Error::success(); +} + /// Process __bug_table section. /// This section contains information useful for kernel debugging. /// Each entry in the section is a struct bug_entry that contains a pointer to @@ -1305,6 +1342,14 @@ Error LinuxKernelRewriter::readAltInstructions() { return Error::success(); } +Error LinuxKernelRewriter::rewriteAltInstructions() { + // Disable output of functions with alt instructions before the rewrite + // support is complete. + skipFunctionsWithAnnotation("AltInst"); + + return Error::success(); +} + /// When the Linux kernel needs to handle an error associated with a given PCI /// device, it uses a table stored in .pci_fixup section to locate a fixup code /// specific to the vendor and the problematic device. The section contains a -- GitLab From 51268a57fd4d7f67fe9fdb337f63ec390fa2379a Mon Sep 17 00:00:00 2001 From: Maksim Panchenko Date: Fri, 22 Mar 2024 15:29:26 -0700 Subject: [PATCH 116/306] [BOLT] Enable --keep-nops option for Linux kernel by default (#86349) Preserve nop instructions in the Linux kernel since they could be used for runtime patching. --- bolt/lib/Rewrite/BinaryPassManager.cpp | 2 +- bolt/lib/Rewrite/RewriteInstance.cpp | 4 ++++ bolt/test/X86/linux-alt-instruction.s | 15 +++++++-------- bolt/test/X86/linux-orc.s | 4 ++-- bolt/test/X86/linux-parainstructions.s | 2 +- 5 files changed, 15 insertions(+), 12 deletions(-) diff --git a/bolt/lib/Rewrite/BinaryPassManager.cpp b/bolt/lib/Rewrite/BinaryPassManager.cpp index 489b33fe1c7c..6c26bb795726 100644 --- a/bolt/lib/Rewrite/BinaryPassManager.cpp +++ b/bolt/lib/Rewrite/BinaryPassManager.cpp @@ -72,7 +72,7 @@ static cl::opt JTFootprintReductionFlag( "instructions at jump sites"), cl::cat(BoltOptCategory)); -static cl::opt +cl::opt KeepNops("keep-nops", cl::desc("keep no-op instructions. By default they are removed."), cl::Hidden, cl::cat(BoltOptCategory)); diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp index 03f4298e817d..2ead51ff6a12 100644 --- a/bolt/lib/Rewrite/RewriteInstance.cpp +++ b/bolt/lib/Rewrite/RewriteInstance.cpp @@ -81,6 +81,7 @@ extern cl::list HotTextMoveSections; extern cl::opt Hugify; extern cl::opt Instrument; extern cl::opt JumpTables; +extern cl::opt KeepNops; extern cl::list ReorderData; extern cl::opt ReorderFunctions; extern cl::opt TimeBuild; @@ -2031,6 +2032,9 @@ void RewriteInstance::adjustCommandLineOptions() { if (opts::Lite) BC->outs() << "BOLT-INFO: enabling lite mode\n"; + + if (BC->IsLinuxKernel && !opts::KeepNops.getNumOccurrences()) + opts::KeepNops = true; } namespace { diff --git a/bolt/test/X86/linux-alt-instruction.s b/bolt/test/X86/linux-alt-instruction.s index 5dcc6fe3ab0c..2cdf31519682 100644 --- a/bolt/test/X86/linux-alt-instruction.s +++ b/bolt/test/X86/linux-alt-instruction.s @@ -6,8 +6,8 @@ # RUN: llvm-mc -filetype=obj -triple x86_64-unknown-unknown %s -o %t.o # RUN: %clang %cflags -nostdlib %t.o -o %t.exe \ # RUN: -Wl,--image-base=0xffffffff80000000,--no-dynamic-linker,--no-eh-frame-hdr,--no-pie -# RUN: llvm-bolt %t.exe --print-normalized --keep-nops -o %t.out \ -# RUN: --alt-inst-feature-size=2 | FileCheck %s +# RUN: llvm-bolt %t.exe --print-normalized --alt-inst-feature-size=2 -o %t.out \ +# RUN: | FileCheck %s ## Older kernels used to have padlen field in alt_instr. Check compatibility. @@ -15,8 +15,8 @@ # RUN: %s -o %t.o # RUN: %clang %cflags -nostdlib %t.o -o %t.exe \ # RUN: -Wl,--image-base=0xffffffff80000000,--no-dynamic-linker,--no-eh-frame-hdr,--no-pie -# RUN: llvm-bolt %t.exe --print-normalized --keep-nops --alt-inst-has-padlen \ -# RUN: -o %t.out | FileCheck %s +# RUN: llvm-bolt %t.exe --print-normalized --alt-inst-has-padlen -o %t.out \ +# RUN: | FileCheck %s ## Check with a larger size of "feature" field in alt_instr. @@ -24,13 +24,12 @@ # RUN: --defsym FEATURE_SIZE_4=1 %s -o %t.o # RUN: %clang %cflags -nostdlib %t.o -o %t.exe \ # RUN: -Wl,--image-base=0xffffffff80000000,--no-dynamic-linker,--no-eh-frame-hdr,--no-pie -# RUN: llvm-bolt %t.exe --print-normalized --keep-nops \ -# RUN: --alt-inst-feature-size=4 -o %t.out | FileCheck %s +# RUN: llvm-bolt %t.exe --print-normalized --alt-inst-feature-size=4 -o %t.out \ +# RUN: | FileCheck %s ## Check that out-of-bounds read is handled properly. -# RUN: not llvm-bolt %t.exe --print-normalized --keep-nops \ -# RUN: --alt-inst-feature-size=2 -o %t.out +# RUN: not llvm-bolt %t.exe --print-normalized --alt-inst-feature-size=2 -o %t.out # CHECK: BOLT-INFO: Linux kernel binary detected # CHECK: BOLT-INFO: parsed 2 alternative instruction entries diff --git a/bolt/test/X86/linux-orc.s b/bolt/test/X86/linux-orc.s index 4da19989408e..5f2096278e92 100644 --- a/bolt/test/X86/linux-orc.s +++ b/bolt/test/X86/linux-orc.s @@ -27,7 +27,7 @@ ## Verify ORC bindings to instructions. # RUN: llvm-bolt %t.exe --print-normalized --dump-orc --print-orc -o %t.out \ -# RUN: --bolt-info=0 |& FileCheck %s +# RUN: --keep-nops=0 --bolt-info=0 |& FileCheck %s ## Verify ORC bindings after rewrite. @@ -37,7 +37,7 @@ ## Verify ORC binding after rewrite when some of the functions are skipped. -# RUN: llvm-bolt %t.exe -o %t.out --skip-funcs=bar --bolt-info=0 +# RUN: llvm-bolt %t.exe -o %t.out --skip-funcs=bar --bolt-info=0 --keep-nops=0 # RUN: llvm-bolt %t.out -o %t.out.1 --print-normalized --print-orc \ # RUN: |& FileCheck %s diff --git a/bolt/test/X86/linux-parainstructions.s b/bolt/test/X86/linux-parainstructions.s index 4bdfde5fb7f2..07fca6bbedaf 100644 --- a/bolt/test/X86/linux-parainstructions.s +++ b/bolt/test/X86/linux-parainstructions.s @@ -8,7 +8,7 @@ ## Verify paravirtual bindings to instructions. -# RUN: llvm-bolt %t.exe --print-normalized -o %t.out | FileCheck %s +# RUN: llvm-bolt %t.exe --print-normalized -o %t.out --keep-nops=0 | FileCheck %s # CHECK: BOLT-INFO: Linux kernel binary detected # CHECK: BOLT-INFO: parsed 2 paravirtual patch sites -- GitLab From 3bc71c2abfa00413fd15cf0e5c08af6ec0d4768b Mon Sep 17 00:00:00 2001 From: Usama Hameed Date: Fri, 22 Mar 2024 15:29:36 -0700 Subject: [PATCH 117/306] Get the linker version and pass the it to compiler-rt tests on Darwin. (#86220) The HOST_LINK_VERSION is a hardcoded string in Darwin clang that detects the linker version at configure time. The driver uses this information to build the correct set of arguments for the linker. This patch detects the linker version again during compiler-rt configuration and passes it to the tests. This allows a clang built on a machine with a new linker to run compiler-rt tests on a machine with an old linker. rdar://125198603 --- clang/CMakeLists.txt | 16 ++-------------- cmake/Modules/GetDarwinLinkerVersion.cmake | 19 +++++++++++++++++++ compiler-rt/CMakeLists.txt | 10 ++++++++++ compiler-rt/test/lit.common.cfg.py | 4 ++++ compiler-rt/test/lit.common.configured.in | 1 + 5 files changed, 36 insertions(+), 14 deletions(-) create mode 100644 cmake/Modules/GetDarwinLinkerVersion.cmake diff --git a/clang/CMakeLists.txt b/clang/CMakeLists.txt index 761dab8c28c1..ee783d52e4a4 100644 --- a/clang/CMakeLists.txt +++ b/clang/CMakeLists.txt @@ -15,6 +15,7 @@ endif() # Must go below project(..) include(GNUInstallDirs) +include(GetDarwinLinkerVersion) if(CLANG_BUILT_STANDALONE) set(CMAKE_CXX_STANDARD 17 CACHE STRING "C++ standard to conform to") @@ -346,20 +347,7 @@ endif () # Determine HOST_LINK_VERSION on Darwin. set(HOST_LINK_VERSION) if (APPLE AND NOT CMAKE_LINKER MATCHES ".*lld.*") - set(LD_V_OUTPUT) - execute_process( - COMMAND sh -c "${CMAKE_LINKER} -v 2>&1 | head -1" - RESULT_VARIABLE HAD_ERROR - OUTPUT_VARIABLE LD_V_OUTPUT - ) - if (HAD_ERROR) - message(FATAL_ERROR "${CMAKE_LINKER} failed with status ${HAD_ERROR}") - endif() - if ("${LD_V_OUTPUT}" MATCHES ".*ld64-([0-9.]+).*") - string(REGEX REPLACE ".*ld64-([0-9.]+).*" "\\1" HOST_LINK_VERSION ${LD_V_OUTPUT}) - elseif ("${LD_V_OUTPUT}" MATCHES "[^0-9]*([0-9.]+).*") - string(REGEX REPLACE "[^0-9]*([0-9.]+).*" "\\1" HOST_LINK_VERSION ${LD_V_OUTPUT}) - endif() + get_darwin_linker_version(HOST_LINK_VERSION) message(STATUS "Host linker version: ${HOST_LINK_VERSION}") endif() diff --git a/cmake/Modules/GetDarwinLinkerVersion.cmake b/cmake/Modules/GetDarwinLinkerVersion.cmake new file mode 100644 index 000000000000..c27e50128586 --- /dev/null +++ b/cmake/Modules/GetDarwinLinkerVersion.cmake @@ -0,0 +1,19 @@ +# Get the linker version on Darwin +function(get_darwin_linker_version variable) + set(LINK_VERSION) + set(LD_V_OUTPUT) + execute_process( + COMMAND sh -c "${CMAKE_LINKER} -v 2>&1 | head -1" + RESULT_VARIABLE HAD_ERROR + OUTPUT_VARIABLE LD_V_OUTPUT + ) + if (HAD_ERROR) + message(FATAL_ERROR "${CMAKE_LINKER} failed with status ${HAD_ERROR}") + endif() + if ("${LD_V_OUTPUT}" MATCHES ".*ld64-([0-9.]+).*") + string(REGEX REPLACE ".*ld64-([0-9.]+).*" "\\1" LINK_VERSION ${LD_V_OUTPUT}) + elseif ("${LD_V_OUTPUT}" MATCHES "[^0-9]*([0-9.]+).*") + string(REGEX REPLACE "[^0-9]*([0-9.]+).*" "\\1" LINK_VERSION ${LD_V_OUTPUT}) + endif() + set(${variable} ${LINK_VERSION} PARENT_SCOPE) +endfunction() diff --git a/compiler-rt/CMakeLists.txt b/compiler-rt/CMakeLists.txt index d562a6206c00..f4e92f14db85 100644 --- a/compiler-rt/CMakeLists.txt +++ b/compiler-rt/CMakeLists.txt @@ -36,6 +36,7 @@ include(SetPlatformToolchainTools) include(base-config-ix) include(CompilerRTUtils) include(CMakeDependentOption) +include(GetDarwinLinkerVersion) option(COMPILER_RT_BUILD_BUILTINS "Build builtins" ON) mark_as_advanced(COMPILER_RT_BUILD_BUILTINS) @@ -444,6 +445,15 @@ else() set(SANITIZER_USE_SYMBOLS FALSE) endif() +# Get the linker version while configuring compiler-rt and explicitly pass it +# in cflags during testing. This fixes the compiler/linker version mismatch +# issue when running a clang built with a newer Xcode in an older Xcode +set(COMPILER_RT_DARWIN_LINKER_VERSION) +if (APPLE AND NOT CMAKE_LINKER MATCHES ".*lld.*") + get_darwin_linker_version(COMPILER_RT_DARWIN_LINKER_VERSION) + message(STATUS "Host linker version: ${COMPILER_RT_DARWIN_LINKER_VERSION}") +endif() + # Build sanitizer runtimes with debug info. if(MSVC) # Use /Z7 instead of /Zi for the asan runtime. This avoids the LNK4099 diff --git a/compiler-rt/test/lit.common.cfg.py b/compiler-rt/test/lit.common.cfg.py index bd9b926c1505..0ac20a9831d9 100644 --- a/compiler-rt/test/lit.common.cfg.py +++ b/compiler-rt/test/lit.common.cfg.py @@ -882,6 +882,10 @@ if config.use_lld and config.has_lld and not config.use_lto: elif config.use_lld and (not config.has_lld): config.unsupported = True +if config.host_os == "Darwin": + if getattr(config, "darwin_linker_version", None): + extra_cflags += ["-mlinker-version=" + config.darwin_linker_version] + # Append any extra flags passed in lit_config append_target_cflags = lit_config.params.get("append_target_cflags", None) if append_target_cflags: diff --git a/compiler-rt/test/lit.common.configured.in b/compiler-rt/test/lit.common.configured.in index db5d7c598b73..fff5dc6cc750 100644 --- a/compiler-rt/test/lit.common.configured.in +++ b/compiler-rt/test/lit.common.configured.in @@ -51,6 +51,7 @@ set_default("expensive_checks", @LLVM_ENABLE_EXPENSIVE_CHECKS_PYBOOL@) set_default("test_standalone_build_libs", @COMPILER_RT_TEST_STANDALONE_BUILD_LIBS_PYBOOL@) set_default("has_compiler_rt_libatomic", @COMPILER_RT_BUILD_STANDALONE_LIBATOMIC_PYBOOL@) set_default("aarch64_sme", @COMPILER_RT_HAS_AARCH64_SME_PYBOOL@) +set_default("darwin_linker_version", "@COMPILER_RT_DARWIN_LINKER_VERSION@") # True iff the test suite supports ignoring the test compiler's runtime library path # and using `config.compiler_rt_libdir` instead. This only matters when the runtime # library paths differ. -- GitLab From 4406e4a8bd5acadd980d84356b36030cadf9a61d Mon Sep 17 00:00:00 2001 From: Muhammad Omair Javaid Date: Sat, 23 Mar 2024 03:22:24 +0500 Subject: [PATCH 118/306] Revert "Missed a null-ptr check in previous PR for Debuginfod testing (#86292)" This reverts commit b1575f9082071702bd6aaa2600ce9fe011a091e9. --- lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp b/lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp index a9956aa9075f..91b8b4a979e0 100644 --- a/lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp +++ b/lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp @@ -53,7 +53,7 @@ static bool IsDwpSymbolFile(const lldb::ModuleSP &module_sp, ObjectFileSP dwp_obj_file = ObjectFile::FindPlugin( module_sp, &file_spec, 0, FileSystem::Instance().GetByteSize(file_spec), dwp_file_data_sp, dwp_file_data_offset); - if (!dwp_obj_file || !ObjectFileELF::classof(dwp_obj_file.get())) + if (!ObjectFileELF::classof(dwp_obj_file.get())) return false; // The presence of a debug_cu_index section is the key identifying feature of // a DWP file. Make sure we don't fill in the section list on dwp_obj_file -- GitLab From 7fc2fbb3f1961e0ad0722c2d749ddd6264195a1c Mon Sep 17 00:00:00 2001 From: Muhammad Omair Javaid Date: Sat, 23 Mar 2024 03:22:33 +0500 Subject: [PATCH 119/306] Revert "DebugInfoD tests + fixing issues exposed by tests (#85693)" This reverts commit 6d939a6ec69adf284cdbef2034b49fd02ba503fc. This broke following LLDB bots: https://lab.llvm.org/buildbot/#/builders/96/builds/54867 https://lab.llvm.org/buildbot/#/builders/17/builds/50824 --- .../Python/lldbsuite/test/make/Makefile.rules | 33 +-- .../SymbolFile/DWARF/SymbolFileDWARF.cpp | 38 ++-- .../Plugins/SymbolLocator/CMakeLists.txt | 7 +- .../SymbolVendor/ELF/SymbolVendorELF.cpp | 30 +-- lldb/test/API/debuginfod/Normal/Makefile | 25 --- .../API/debuginfod/Normal/TestDebuginfod.py | 185 ----------------- lldb/test/API/debuginfod/Normal/main.c | 7 - lldb/test/API/debuginfod/SplitDWARF/Makefile | 28 --- .../SplitDWARF/TestDebuginfodDWP.py | 194 ------------------ lldb/test/API/debuginfod/SplitDWARF/main.c | 7 - 10 files changed, 17 insertions(+), 537 deletions(-) delete mode 100644 lldb/test/API/debuginfod/Normal/Makefile delete mode 100644 lldb/test/API/debuginfod/Normal/TestDebuginfod.py delete mode 100644 lldb/test/API/debuginfod/Normal/main.c delete mode 100644 lldb/test/API/debuginfod/SplitDWARF/Makefile delete mode 100644 lldb/test/API/debuginfod/SplitDWARF/TestDebuginfodDWP.py delete mode 100644 lldb/test/API/debuginfod/SplitDWARF/main.c diff --git a/lldb/packages/Python/lldbsuite/test/make/Makefile.rules b/lldb/packages/Python/lldbsuite/test/make/Makefile.rules index 75efcde1f040..bfd249ccd43f 100644 --- a/lldb/packages/Python/lldbsuite/test/make/Makefile.rules +++ b/lldb/packages/Python/lldbsuite/test/make/Makefile.rules @@ -51,7 +51,7 @@ LLDB_BASE_DIR := $(THIS_FILE_DIR)/../../../../../ # # GNUWin32 uname gives "windows32" or "server version windows32" while # some versions of MSYS uname return "MSYS_NT*", but most environments -# standardize on "Windows_NT", so we'll make it consistent here. +# standardize on "Windows_NT", so we'll make it consistent here. # When running tests from Visual Studio, the environment variable isn't # inherited all the way down to the process spawned for make. #---------------------------------------------------------------------- @@ -210,12 +210,6 @@ else ifeq "$(SPLIT_DEBUG_SYMBOLS)" "YES" DSYM = $(EXE).debug endif - - ifeq "$(MAKE_DWP)" "YES" - MAKE_DWO := YES - DWP_NAME = $(EXE).dwp - DYLIB_DWP_NAME = $(DYLIB_NAME).dwp - endif endif LIMIT_DEBUG_INFO_FLAGS = @@ -363,7 +357,6 @@ ifneq "$(OS)" "Darwin" OBJCOPY ?= $(call replace_cc_with,objcopy) ARCHIVER ?= $(call replace_cc_with,ar) - DWP ?= $(call replace_cc_with,dwp) override AR = $(ARCHIVER) endif @@ -534,10 +527,6 @@ ifneq "$(CXX)" "" endif endif -ifeq "$(GEN_GNU_BUILD_ID)" "YES" - LDFLAGS += -Wl,--build-id -endif - #---------------------------------------------------------------------- # DYLIB_ONLY variable can be used to skip the building of a.out. # See the sections below regarding dSYM file as well as the building of @@ -576,25 +565,11 @@ else endif else ifeq "$(SPLIT_DEBUG_SYMBOLS)" "YES" -ifeq "$(SAVE_FULL_DEBUG_BINARY)" "YES" - cp "$(EXE)" "$(EXE).full" -endif $(OBJCOPY) --only-keep-debug "$(EXE)" "$(DSYM)" $(OBJCOPY) --strip-debug --add-gnu-debuglink="$(DSYM)" "$(EXE)" "$(EXE)" endif -ifeq "$(MAKE_DWP)" "YES" - $(DWP) -o "$(DWP_NAME)" $(DWOS) -endif endif - -#---------------------------------------------------------------------- -# Support emitting the content of the GNU build-id into a file -# This needs to be used in conjunction with GEN_GNU_BUILD_ID := YES -#---------------------------------------------------------------------- -$(EXE).uuid : $(EXE) - $(OBJCOPY) --dump-section=.note.gnu.build-id=$@ $< - #---------------------------------------------------------------------- # Make the dylib #---------------------------------------------------------------------- @@ -635,15 +610,9 @@ endif else $(LD) $(DYLIB_OBJECTS) $(LDFLAGS) -shared -o "$(DYLIB_FILENAME)" ifeq "$(SPLIT_DEBUG_SYMBOLS)" "YES" - ifeq "$(SAVE_FULL_DEBUG_BINARY)" "YES" - cp "$(DYLIB_FILENAME)" "$(DYLIB_FILENAME).full" - endif $(OBJCOPY) --only-keep-debug "$(DYLIB_FILENAME)" "$(DYLIB_FILENAME).debug" $(OBJCOPY) --strip-debug --add-gnu-debuglink="$(DYLIB_FILENAME).debug" "$(DYLIB_FILENAME)" "$(DYLIB_FILENAME)" endif -ifeq "$(MAKE_DWP)" "YES" - $(DWP) -o $(DYLIB_DWP_FILE) $(DYLIB_DWOS) -endif endif #---------------------------------------------------------------------- diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp index 08ce7b82b0c1..5f67658f86ea 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp +++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp @@ -4377,38 +4377,26 @@ const std::shared_ptr &SymbolFileDWARF::GetDwpSymbolFile() { FileSpecList search_paths = Target::GetDefaultDebugFileSearchPaths(); ModuleSpec module_spec; module_spec.GetFileSpec() = m_objfile_sp->GetFileSpec(); - FileSpec dwp_filespec; for (const auto &symfile : symfiles.files()) { module_spec.GetSymbolFileSpec() = FileSpec(symfile.GetPath() + ".dwp", symfile.GetPathStyle()); LLDB_LOG(log, "Searching for DWP using: \"{0}\"", module_spec.GetSymbolFileSpec()); - dwp_filespec = + FileSpec dwp_filespec = PluginManager::LocateExecutableSymbolFile(module_spec, search_paths); if (FileSystem::Instance().Exists(dwp_filespec)) { - break; - } - } - if (!FileSystem::Instance().Exists(dwp_filespec)) { - LLDB_LOG(log, "No DWP file found locally"); - // Fill in the UUID for the module we're trying to match for, so we can - // find the correct DWP file, as the Debuginfod plugin uses *only* this - // data to correctly match the DWP file with the binary. - module_spec.GetUUID() = m_objfile_sp->GetUUID(); - dwp_filespec = - PluginManager::LocateExecutableSymbolFile(module_spec, search_paths); - } - if (FileSystem::Instance().Exists(dwp_filespec)) { - LLDB_LOG(log, "Found DWP file: \"{0}\"", dwp_filespec); - DataBufferSP dwp_file_data_sp; - lldb::offset_t dwp_file_data_offset = 0; - ObjectFileSP dwp_obj_file = ObjectFile::FindPlugin( - GetObjectFile()->GetModule(), &dwp_filespec, 0, - FileSystem::Instance().GetByteSize(dwp_filespec), dwp_file_data_sp, - dwp_file_data_offset); - if (dwp_obj_file) { - m_dwp_symfile = std::make_shared( - *this, dwp_obj_file, DIERef::k_file_index_mask); + LLDB_LOG(log, "Found DWP file: \"{0}\"", dwp_filespec); + DataBufferSP dwp_file_data_sp; + lldb::offset_t dwp_file_data_offset = 0; + ObjectFileSP dwp_obj_file = ObjectFile::FindPlugin( + GetObjectFile()->GetModule(), &dwp_filespec, 0, + FileSystem::Instance().GetByteSize(dwp_filespec), dwp_file_data_sp, + dwp_file_data_offset); + if (dwp_obj_file) { + m_dwp_symfile = std::make_shared( + *this, dwp_obj_file, DIERef::k_file_index_mask); + break; + } } } if (!m_dwp_symfile) { diff --git a/lldb/source/Plugins/SymbolLocator/CMakeLists.txt b/lldb/source/Plugins/SymbolLocator/CMakeLists.txt index 3367022639ab..ca969626f4ff 100644 --- a/lldb/source/Plugins/SymbolLocator/CMakeLists.txt +++ b/lldb/source/Plugins/SymbolLocator/CMakeLists.txt @@ -1,10 +1,5 @@ -# Order matters here: the first symbol locator prevents further searching. -# For DWARF binaries that are both stripped and split, the Default plugin -# will return the stripped binary when asked for the ObjectFile, which then -# prevents an unstripped binary from being requested from the Debuginfod -# provider. -add_subdirectory(Debuginfod) add_subdirectory(Default) if (CMAKE_SYSTEM_NAME MATCHES "Darwin") add_subdirectory(DebugSymbols) endif() +add_subdirectory(Debuginfod) diff --git a/lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp b/lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp index 91b8b4a979e0..b5fe35d71032 100644 --- a/lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp +++ b/lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp @@ -44,25 +44,6 @@ llvm::StringRef SymbolVendorELF::GetPluginDescriptionStatic() { "executables."; } -// If this is needed elsewhere, it can be exported/moved. -static bool IsDwpSymbolFile(const lldb::ModuleSP &module_sp, - const FileSpec &file_spec) { - DataBufferSP dwp_file_data_sp; - lldb::offset_t dwp_file_data_offset = 0; - // Try to create an ObjectFile from the file_spec. - ObjectFileSP dwp_obj_file = ObjectFile::FindPlugin( - module_sp, &file_spec, 0, FileSystem::Instance().GetByteSize(file_spec), - dwp_file_data_sp, dwp_file_data_offset); - if (!ObjectFileELF::classof(dwp_obj_file.get())) - return false; - // The presence of a debug_cu_index section is the key identifying feature of - // a DWP file. Make sure we don't fill in the section list on dwp_obj_file - // (by calling GetSectionList(false)) as this is invoked before we may have - // all the symbol files collected and available. - return dwp_obj_file && dwp_obj_file->GetSectionList(false)->FindSectionByType( - eSectionTypeDWARFDebugCuIndex, false); -} - // CreateInstance // // Platforms can register a callback to use when creating symbol vendors to @@ -106,15 +87,8 @@ SymbolVendorELF::CreateInstance(const lldb::ModuleSP &module_sp, FileSpecList search_paths = Target::GetDefaultDebugFileSearchPaths(); FileSpec dsym_fspec = PluginManager::LocateExecutableSymbolFile(module_spec, search_paths); - if (!dsym_fspec || IsDwpSymbolFile(module_sp, dsym_fspec)) { - // If we have a stripped binary or if we got a DWP file, we should prefer - // symbols in the executable acquired through a plugin. - ModuleSpec unstripped_spec = - PluginManager::LocateExecutableObjectFile(module_spec); - if (!unstripped_spec) - return nullptr; - dsym_fspec = unstripped_spec.GetFileSpec(); - } + if (!dsym_fspec) + return nullptr; DataBufferSP dsym_file_data_sp; lldb::offset_t dsym_file_data_offset = 0; diff --git a/lldb/test/API/debuginfod/Normal/Makefile b/lldb/test/API/debuginfod/Normal/Makefile deleted file mode 100644 index bd2fa623df47..000000000000 --- a/lldb/test/API/debuginfod/Normal/Makefile +++ /dev/null @@ -1,25 +0,0 @@ -C_SOURCES := main.c - -# For normal (non DWP) Debuginfod tests, we need: - -# * The "full" binary: a.out.debug -# Produced by Makefile.rules with KEEP_FULL_DEBUG_BINARY set to YES and -# SPLIT_DEBUG_SYMBOLS set to YES - -# * The stripped binary (a.out) -# Produced by Makefile.rules with SPLIT_DEBUG_SYMBOLS set to YES - -# * The 'only-keep-debug' binary (a.out.dbg) -# Produced below - -# * The .uuid file (for a little easier testing code) -# Produced below - -# Don't strip the debug info from a.out: -SPLIT_DEBUG_SYMBOLS := YES -SAVE_FULL_DEBUG_BINARY := YES -GEN_GNU_BUILD_ID := YES - -all: a.out.uuid a.out - -include Makefile.rules diff --git a/lldb/test/API/debuginfod/Normal/TestDebuginfod.py b/lldb/test/API/debuginfod/Normal/TestDebuginfod.py deleted file mode 100644 index eb5efe83c17a..000000000000 --- a/lldb/test/API/debuginfod/Normal/TestDebuginfod.py +++ /dev/null @@ -1,185 +0,0 @@ -import os -import shutil -import tempfile -import struct - -import lldb -from lldbsuite.test.decorators import * -import lldbsuite.test.lldbutil as lldbutil -from lldbsuite.test.lldbtest import * - - -def getUUID(aoutuuid): - """ - Pull the 20 byte UUID out of the .note.gnu.build-id section that was dumped - to a file already, as part of the build. - """ - with open(aoutuuid, "rb") as f: - data = f.read(36) - if len(data) != 36: - return None - header = struct.unpack_from("<4I", data) - if len(header) != 4: - return None - # 4 element 'prefix', 20 bytes of uuid, 3 byte long string: 'GNU': - if header[0] != 4 or header[1] != 20 or header[2] != 3 or header[3] != 0x554E47: - return None - return data[16:].hex() - - -""" -Test support for the DebugInfoD network symbol acquisition protocol. -This one is for simple / no split-dwarf scenarios. - -For no-split-dwarf scenarios, there are 2 variations: -1 - A stripped binary with it's corresponding unstripped binary: -2 - A stripped binary with a corresponding --only-keep-debug symbols file -""" - - -@skipUnlessPlatform(["linux", "freebsd"]) -class DebugInfodTests(TestBase): - # No need to try every flavor of debug inf. - NO_DEBUG_INFO_TESTCASE = True - - def test_normal_no_symbols(self): - """ - Validate behavior with no symbols or symbol locator. - ('baseline negative' behavior) - """ - test_root = self.config_test(["a.out"]) - self.try_breakpoint(False) - - def test_normal_default(self): - """ - Validate behavior with symbols, but no symbol locator. - ('baseline positive' behavior) - """ - test_root = self.config_test(["a.out", "a.out.debug"]) - self.try_breakpoint(True) - - def test_debuginfod_symbols(self): - """ - Test behavior with the full binary available from Debuginfod as - 'debuginfo' from the plug-in. - """ - test_root = self.config_test(["a.out"], "a.out.full") - self.try_breakpoint(True) - - def test_debuginfod_executable(self): - """ - Test behavior with the full binary available from Debuginfod as - 'executable' from the plug-in. - """ - test_root = self.config_test(["a.out"], None, "a.out.full") - self.try_breakpoint(True) - - def test_debuginfod_okd_symbols(self): - """ - Test behavior with the 'only-keep-debug' symbols available from Debuginfod. - """ - test_root = self.config_test(["a.out"], "a.out.debug") - self.try_breakpoint(True) - - def try_breakpoint(self, should_have_loc): - """ - This function creates a target from self.aout, sets a function-name - breakpoint, and checks to see if we have a file/line location, - as a way to validate that the symbols have been loaded. - should_have_loc specifies if we're testing that symbols have or - haven't been loaded. - """ - target = self.dbg.CreateTarget(self.aout) - self.assertTrue(target and target.IsValid(), "Target is valid") - - bp = target.BreakpointCreateByName("func") - self.assertTrue(bp and bp.IsValid(), "Breakpoint is valid") - self.assertEqual(bp.GetNumLocations(), 1) - - loc = bp.GetLocationAtIndex(0) - self.assertTrue(loc and loc.IsValid(), "Location is valid") - addr = loc.GetAddress() - self.assertTrue(addr and addr.IsValid(), "Loc address is valid") - line_entry = addr.GetLineEntry() - self.assertEqual( - should_have_loc, - line_entry != None and line_entry.IsValid(), - "Loc line entry is valid", - ) - if should_have_loc: - self.assertEqual(line_entry.GetLine(), 4) - self.assertEqual( - line_entry.GetFileSpec().GetFilename(), - self.main_source_file.GetFilename(), - ) - self.dbg.DeleteTarget(target) - shutil.rmtree(self.tmp_dir) - - def config_test(self, local_files, debuginfo=None, executable=None): - """ - Set up a test with local_files[] copied to a different location - so that we control which files are, or are not, found in the file system. - Also, create a stand-alone file-system 'hosted' debuginfod server with the - provided debuginfo and executable files (if they exist) - - Make the filesystem look like: - - /tmp//test/[local_files] - - /tmp//cache (for lldb to use as a temp cache) - - /tmp//buildid//executable -> - /tmp//buildid//debuginfo -> - Returns the /tmp/ path - """ - - self.build() - - uuid = getUUID(self.getBuildArtifact("a.out.uuid")) - - self.main_source_file = lldb.SBFileSpec("main.c") - self.tmp_dir = tempfile.mkdtemp() - test_dir = os.path.join(self.tmp_dir, "test") - os.makedirs(test_dir) - - self.aout = "" - # Copy the files used by the test: - for f in local_files: - shutil.copy(self.getBuildArtifact(f), test_dir) - # The first item is the binary to be used for the test - if self.aout == "": - self.aout = os.path.join(test_dir, f) - - use_debuginfod = debuginfo != None or executable != None - - # Populated the 'file://... mocked' Debuginfod server: - if use_debuginfod: - os.makedirs(os.path.join(self.tmp_dir, "cache")) - uuid_dir = os.path.join(self.tmp_dir, "buildid", uuid) - os.makedirs(uuid_dir) - if debuginfo: - shutil.copy( - self.getBuildArtifact(debuginfo), - os.path.join(uuid_dir, "debuginfo"), - ) - if executable: - shutil.copy( - self.getBuildArtifact(executable), - os.path.join(uuid_dir, "executable"), - ) - - # Configure LLDB for the test: - self.runCmd( - "settings set symbols.enable-external-lookup %s" - % str(use_debuginfod).lower() - ) - self.runCmd("settings clear plugin.symbol-locator.debuginfod.server-urls") - if use_debuginfod: - self.runCmd( - "settings set plugin.symbol-locator.debuginfod.cache-path %s/cache" - % self.tmp_dir - ) - self.runCmd( - "settings insert-before plugin.symbol-locator.debuginfod.server-urls 0 file://%s" - % self.tmp_dir - ) diff --git a/lldb/test/API/debuginfod/Normal/main.c b/lldb/test/API/debuginfod/Normal/main.c deleted file mode 100644 index 4c7184609b45..000000000000 --- a/lldb/test/API/debuginfod/Normal/main.c +++ /dev/null @@ -1,7 +0,0 @@ -// This is a dump little pair of test files - -int func(int argc, const char *argv[]) { - return (argc + 1) * (argv[argc][0] + 2); -} - -int main(int argc, const char *argv[]) { return func(0, argv); } diff --git a/lldb/test/API/debuginfod/SplitDWARF/Makefile b/lldb/test/API/debuginfod/SplitDWARF/Makefile deleted file mode 100644 index 266d74cf9062..000000000000 --- a/lldb/test/API/debuginfod/SplitDWARF/Makefile +++ /dev/null @@ -1,28 +0,0 @@ -C_SOURCES := main.c - -# For split-dwarf Debuginfod tests, we need: - -# * A .DWP file (a.out.dwp) -# Produced by Makefile.rules with MAKE_DWO and MERGE_DWOS both set to YES - -# * The "full" binary: it's missing things that live in .dwo's (a.out.debug) -# Produced by Makefile.rules with KEEP_FULL_DEBUG_BINARY set to YES and -# SPLIT_DEBUG_SYMBOLS set to YES - -# * The stripped binary (a.out) -# Produced by Makefile.rules - -# * The 'only-keep-debug' binary (a.out.dbg) -# Produced below - -# * The .uuid file (for a little easier testing code) -# Produced here in the rule below - -MAKE_DWP := YES -SPLIT_DEBUG_SYMBOLS := YES -SAVE_FULL_DEBUG_BINARY := YES -GEN_GNU_BUILD_ID := YES - -all: a.out.uuid a.out - -include Makefile.rules diff --git a/lldb/test/API/debuginfod/SplitDWARF/TestDebuginfodDWP.py b/lldb/test/API/debuginfod/SplitDWARF/TestDebuginfodDWP.py deleted file mode 100644 index 09f91b6f1c6c..000000000000 --- a/lldb/test/API/debuginfod/SplitDWARF/TestDebuginfodDWP.py +++ /dev/null @@ -1,194 +0,0 @@ -""" -Test support for the DebugInfoD network symbol acquisition protocol. -""" -import os -import shutil -import tempfile -import struct - -import lldb -from lldbsuite.test.decorators import * -import lldbsuite.test.lldbutil as lldbutil -from lldbsuite.test.lldbtest import * - - -def getUUID(aoutuuid): - """ - Pull the 20 byte UUID out of the .note.gnu.build-id section that was dumped - to a file already, as part of the build. - """ - with open(aoutuuid, "rb") as f: - data = f.read(36) - if len(data) != 36: - return None - header = struct.unpack_from("<4I", data) - if len(header) != 4: - return None - # 4 element 'prefix', 20 bytes of uuid, 3 byte long string: 'GNU': - if header[0] != 4 or header[1] != 20 or header[2] != 3 or header[3] != 0x554E47: - return None - return data[16:].hex() - - -""" -Test support for the DebugInfoD network symbol acquisition protocol. -This file is for split-dwarf (dwp) scenarios. - -1 - A split binary target with it's corresponding DWP file -2 - A stripped, split binary target with an unstripped binary and a DWP file -3 - A stripped, split binary target with an --only-keep-debug symbols file and a DWP file -""" - - -@skipUnlessPlatform(["linux", "freebsd"]) -class DebugInfodDWPTests(TestBase): - # No need to try every flavor of debug inf. - NO_DEBUG_INFO_TESTCASE = True - - def test_normal_stripped(self): - """ - Validate behavior with a stripped binary, no symbols or symbol locator. - """ - self.config_test(["a.out"]) - self.try_breakpoint(False) - - def test_normal_stripped_split_with_dwp(self): - """ - Validate behavior with symbols, but no symbol locator. - """ - self.config_test(["a.out", "a.out.debug", "a.out.dwp"]) - self.try_breakpoint(True) - - def test_normal_stripped_only_dwp(self): - """ - Validate behavior *with* dwp symbols only, but missing other symbols, - but no symbol locator. This shouldn't work: without the other symbols - DWO's appear mostly useless. - """ - self.config_test(["a.out", "a.out.dwp"]) - self.try_breakpoint(False) - - def test_debuginfod_dwp_from_service(self): - """ - Test behavior with the unstripped binary, and DWP from the service. - """ - self.config_test(["a.out.debug"], "a.out.dwp") - self.try_breakpoint(True) - - def test_debuginfod_both_symfiles_from_service(self): - """ - Test behavior with a stripped binary, with the unstripped binary and - dwp symbols from Debuginfod. - """ - self.config_test(["a.out"], "a.out.dwp", "a.out.full") - self.try_breakpoint(True) - - def test_debuginfod_both_okd_symfiles_from_service(self): - """ - Test behavior with both the only-keep-debug symbols and the dwp symbols - from Debuginfod. - """ - self.config_test(["a.out"], "a.out.dwp", "a.out.debug") - self.try_breakpoint(True) - - def try_breakpoint(self, should_have_loc): - """ - This function creates a target from self.aout, sets a function-name - breakpoint, and checks to see if we have a file/line location, - as a way to validate that the symbols have been loaded. - should_have_loc specifies if we're testing that symbols have or - haven't been loaded. - """ - target = self.dbg.CreateTarget(self.aout) - self.assertTrue(target and target.IsValid(), "Target is valid") - - bp = target.BreakpointCreateByName("func") - self.assertTrue(bp and bp.IsValid(), "Breakpoint is valid") - self.assertEqual(bp.GetNumLocations(), 1) - - loc = bp.GetLocationAtIndex(0) - self.assertTrue(loc and loc.IsValid(), "Location is valid") - addr = loc.GetAddress() - self.assertTrue(addr and addr.IsValid(), "Loc address is valid") - line_entry = addr.GetLineEntry() - self.assertEqual( - should_have_loc, - line_entry != None and line_entry.IsValid(), - "Loc line entry is valid", - ) - if should_have_loc: - self.assertEqual(line_entry.GetLine(), 4) - self.assertEqual( - line_entry.GetFileSpec().GetFilename(), - self.main_source_file.GetFilename(), - ) - self.dbg.DeleteTarget(target) - shutil.rmtree(self.tmp_dir) - - def config_test(self, local_files, debuginfo=None, executable=None): - """ - Set up a test with local_files[] copied to a different location - so that we control which files are, or are not, found in the file system. - Also, create a stand-alone file-system 'hosted' debuginfod server with the - provided debuginfo and executable files (if they exist) - - Make the filesystem look like: - - /tmp//test/[local_files] - - /tmp//cache (for lldb to use as a temp cache) - - /tmp//buildid//executable -> - /tmp//buildid//debuginfo -> - Returns the /tmp/ path - """ - - self.build() - - uuid = getUUID(self.getBuildArtifact("a.out.uuid")) - - self.main_source_file = lldb.SBFileSpec("main.c") - self.tmp_dir = tempfile.mkdtemp() - self.test_dir = os.path.join(self.tmp_dir, "test") - os.makedirs(self.test_dir) - - self.aout = "" - # Copy the files used by the test: - for f in local_files: - shutil.copy(self.getBuildArtifact(f), self.test_dir) - if self.aout == "": - self.aout = os.path.join(self.test_dir, f) - - use_debuginfod = debuginfo != None or executable != None - - # Populated the 'file://... mocked' Debuginfod server: - if use_debuginfod: - os.makedirs(os.path.join(self.tmp_dir, "cache")) - uuid_dir = os.path.join(self.tmp_dir, "buildid", uuid) - os.makedirs(uuid_dir) - if debuginfo: - shutil.copy( - self.getBuildArtifact(debuginfo), - os.path.join(uuid_dir, "debuginfo"), - ) - if executable: - shutil.copy( - self.getBuildArtifact(executable), - os.path.join(uuid_dir, "executable"), - ) - os.remove(self.getBuildArtifact("main.dwo")) - # Configure LLDB for the test: - self.runCmd( - "settings set symbols.enable-external-lookup %s" - % str(use_debuginfod).lower() - ) - self.runCmd("settings clear plugin.symbol-locator.debuginfod.server-urls") - if use_debuginfod: - self.runCmd( - "settings set plugin.symbol-locator.debuginfod.cache-path %s/cache" - % self.tmp_dir - ) - self.runCmd( - "settings insert-before plugin.symbol-locator.debuginfod.server-urls 0 file://%s" - % self.tmp_dir - ) diff --git a/lldb/test/API/debuginfod/SplitDWARF/main.c b/lldb/test/API/debuginfod/SplitDWARF/main.c deleted file mode 100644 index 4c7184609b45..000000000000 --- a/lldb/test/API/debuginfod/SplitDWARF/main.c +++ /dev/null @@ -1,7 +0,0 @@ -// This is a dump little pair of test files - -int func(int argc, const char *argv[]) { - return (argc + 1) * (argv[argc][0] + 2); -} - -int main(int argc, const char *argv[]) { return func(0, argv); } -- GitLab From 00248754176d74aed2e0785d9982a5ea8e91a71a Mon Sep 17 00:00:00 2001 From: "Daniel M. Katz" Date: Fri, 22 Mar 2024 19:08:02 -0400 Subject: [PATCH 120/306] [Clang] Raise an error on namespace aliases with qualified names. (#86122) --- clang/docs/ReleaseNotes.rst | 2 ++ clang/include/clang/Basic/DiagnosticParseKinds.td | 2 ++ clang/lib/Parse/ParseDeclCXX.cpp | 8 ++++++++ clang/test/SemaCXX/namespace-alias.cpp | 2 ++ 4 files changed, 14 insertions(+) diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index d6e179ca9d69..8054d90fc70f 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -437,6 +437,8 @@ Bug Fixes to C++ Support - Clang's __builtin_bit_cast will now produce a constant value for records with empty bases. See: (#GH82383) - Fix a crash when instantiating a lambda that captures ``this`` outside of its context. Fixes (#GH85343). +- Fix an issue where a namespace alias could be defined using a qualified name (all name components + following the first `::` were ignored). Bug Fixes to AST Handling ^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/clang/include/clang/Basic/DiagnosticParseKinds.td b/clang/include/clang/Basic/DiagnosticParseKinds.td index 48de5e2ef5f4..46a44418a315 100644 --- a/clang/include/clang/Basic/DiagnosticParseKinds.td +++ b/clang/include/clang/Basic/DiagnosticParseKinds.td @@ -268,6 +268,8 @@ def err_expected_semi_after_namespace_name : Error< "expected ';' after namespace name">; def err_unexpected_namespace_attributes_alias : Error< "attributes cannot be specified on namespace alias">; +def err_unexpected_qualified_namespace_alias : Error< + "namespace alias must be a single identifier">; def err_unexpected_nested_namespace_attribute : Error< "attributes cannot be specified on a nested namespace definition">; def err_inline_namespace_alias : Error<"namespace alias cannot be inline">; diff --git a/clang/lib/Parse/ParseDeclCXX.cpp b/clang/lib/Parse/ParseDeclCXX.cpp index 77d2382ea6d9..63fe678cbb29 100644 --- a/clang/lib/Parse/ParseDeclCXX.cpp +++ b/clang/lib/Parse/ParseDeclCXX.cpp @@ -140,6 +140,14 @@ Parser::DeclGroupPtrTy Parser::ParseNamespace(DeclaratorContext Context, SkipUntil(tok::semi); return nullptr; } + if (!ExtraNSs.empty()) { + Diag(ExtraNSs.front().NamespaceLoc, + diag::err_unexpected_qualified_namespace_alias) + << SourceRange(ExtraNSs.front().NamespaceLoc, + ExtraNSs.back().IdentLoc); + SkipUntil(tok::semi); + return nullptr; + } if (attrLoc.isValid()) Diag(attrLoc, diag::err_unexpected_namespace_attributes_alias); if (InlineLoc.isValid()) diff --git a/clang/test/SemaCXX/namespace-alias.cpp b/clang/test/SemaCXX/namespace-alias.cpp index 281ee9962e8b..591957a657c0 100644 --- a/clang/test/SemaCXX/namespace-alias.cpp +++ b/clang/test/SemaCXX/namespace-alias.cpp @@ -47,6 +47,8 @@ namespace I { namespace A1 { int i; } namespace A2 = A1; + + namespace A3::extra::specifiers = A2; // expected-error {{alias must be a single identifier}} } int f() { -- GitLab From 5d187898f625cc54310f51b278b36ad6a97104ee Mon Sep 17 00:00:00 2001 From: Fabian Tschopp Date: Sat, 23 Mar 2024 00:09:11 +0100 Subject: [PATCH 121/306] [mlir][inliner] Return early if the inliningThreshold is 0U or -1U. (#86287) Computing the inlinling profitability can be costly due to walking the graph when counting the number of operations. This PR addresses that by returning early if the threshold is set to never or always inline. --- mlir/lib/Transforms/InlinerPass.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/mlir/lib/Transforms/InlinerPass.cpp b/mlir/lib/Transforms/InlinerPass.cpp index 08d8dbf73a6a..9a7d5403a95d 100644 --- a/mlir/lib/Transforms/InlinerPass.cpp +++ b/mlir/lib/Transforms/InlinerPass.cpp @@ -93,12 +93,19 @@ InlinerPass::InlinerPass(std::function defaultPipeline, // Return true if the inlining ratio does not exceed the threshold. static bool isProfitableToInline(const Inliner::ResolvedCall &resolvedCall, unsigned inliningThreshold) { + // Return early, ratio <= 0U will always be false. + if (inliningThreshold == 0U) + return false; + // Return early, ratio <= -1U will always be true. + if (inliningThreshold == -1U) + return true; + Region *callerRegion = resolvedCall.sourceNode->getCallableRegion(); Region *calleeRegion = resolvedCall.targetNode->getCallableRegion(); // We should not get external nodes here, but just return true // for now to preserve the original behavior of the inliner pass. - if (!calleeRegion || !calleeRegion) + if (!callerRegion || !calleeRegion) return true; auto countOps = [](Region *region) { -- GitLab From b768a8c1db85b9e84fd8b356570a3a8fbe37acf6 Mon Sep 17 00:00:00 2001 From: Jan Svoboda Date: Fri, 22 Mar 2024 16:09:34 -0700 Subject: [PATCH 122/306] [clang][deps] Lazy dependency directives (#86347) Since b4c83a13f664582015ea22924b9a0c6290d41f5b, `Preprocessor` and `Lexer` are aware of the concept of scanning dependency directives. This makes it possible to scan for them on-demand rather than eagerly on the first filesystem operation (open, or even just stat). This might improve performance, but is also necessary for the "PCH as module" mode. Some precompiled header sources use the ".pch" file extension, which means they were not getting scanned for dependency directives. This was okay when the PCH was the main input file in a separate scan step, because there we just lex the file in a scanning-specific frontend action. But when such source gets treated as a module implicitly loaded from a TU, it will get compiled as any other module - with Sema - which will result in compilation errors. (See attached test case.) rdar://107663951 --- .../DependencyScanningFilesystem.h | 20 +++--- .../DependencyScanningFilesystem.cpp | 63 +++++-------------- .../DependencyScanningWorker.cpp | 3 +- clang/test/ClangScanDeps/modules-extension.c | 33 ++++++++++ 4 files changed, 60 insertions(+), 59 deletions(-) create mode 100644 clang/test/ClangScanDeps/modules-extension.c diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h index 846fdc725397..9a522a3e2fe2 100644 --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h @@ -242,6 +242,8 @@ class EntryRef { /// The underlying cached entry. const CachedFileSystemEntry &Entry; + friend class DependencyScanningWorkerFilesystem; + public: EntryRef(StringRef Name, const CachedFileSystemEntry &Entry) : Filename(Name), Entry(Entry) {} @@ -300,14 +302,15 @@ public: /// /// Attempts to use the local and shared caches first, then falls back to /// using the underlying filesystem. - llvm::ErrorOr - getOrCreateFileSystemEntry(StringRef Filename, - bool DisableDirectivesScanning = false); + llvm::ErrorOr getOrCreateFileSystemEntry(StringRef Filename); -private: - /// Check whether the file should be scanned for preprocessor directives. - bool shouldScanForDirectives(StringRef Filename); + /// Ensure the directive tokens are populated for this file entry. + /// + /// Returns true if the directive tokens are populated for this file entry, + /// false if not (i.e. this entry is not a file or its scan fails). + bool ensureDirectiveTokensArePopulated(EntryRef Entry); +private: /// For a filename that's not yet associated with any entry in the caches, /// uses the underlying filesystem to either look up the entry based in the /// shared cache indexed by unique ID, or creates new entry from scratch. @@ -317,11 +320,6 @@ private: computeAndStoreResult(StringRef OriginalFilename, StringRef FilenameForLookup); - /// Scan for preprocessor directives for the given entry if necessary and - /// returns a wrapper object with reference semantics. - EntryRef scanForDirectivesIfNecessary(const CachedFileSystemEntry &Entry, - StringRef Filename, bool Disable); - /// Represents a filesystem entry that has been stat-ed (and potentially read) /// and that's about to be inserted into the cache as `CachedFileSystemEntry`. struct TentativeEntry { diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp index 1b750cec41e1..9b7812a1adb9 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp @@ -41,24 +41,25 @@ DependencyScanningWorkerFilesystem::readFile(StringRef Filename) { return TentativeEntry(Stat, std::move(Buffer)); } -EntryRef DependencyScanningWorkerFilesystem::scanForDirectivesIfNecessary( - const CachedFileSystemEntry &Entry, StringRef Filename, bool Disable) { - if (Entry.isError() || Entry.isDirectory() || Disable || - !shouldScanForDirectives(Filename)) - return EntryRef(Filename, Entry); +bool DependencyScanningWorkerFilesystem::ensureDirectiveTokensArePopulated( + EntryRef Ref) { + auto &Entry = Ref.Entry; + + if (Entry.isError() || Entry.isDirectory()) + return false; CachedFileContents *Contents = Entry.getCachedContents(); assert(Contents && "contents not initialized"); // Double-checked locking. if (Contents->DepDirectives.load()) - return EntryRef(Filename, Entry); + return true; std::lock_guard GuardLock(Contents->ValueLock); // Double-checked locking. if (Contents->DepDirectives.load()) - return EntryRef(Filename, Entry); + return true; SmallVector Directives; // Scan the file for preprocessor directives that might affect the @@ -69,16 +70,16 @@ EntryRef DependencyScanningWorkerFilesystem::scanForDirectivesIfNecessary( Contents->DepDirectiveTokens.clear(); // FIXME: Propagate the diagnostic if desired by the client. Contents->DepDirectives.store(new std::optional()); - return EntryRef(Filename, Entry); + return false; } // This function performed double-checked locking using `DepDirectives`. // Assigning it must be the last thing this function does, otherwise other - // threads may skip the - // critical section (`DepDirectives != nullptr`), leading to a data race. + // threads may skip the critical section (`DepDirectives != nullptr`), leading + // to a data race. Contents->DepDirectives.store( new std::optional(std::move(Directives))); - return EntryRef(Filename, Entry); + return true; } DependencyScanningFilesystemSharedCache:: @@ -161,34 +162,11 @@ DependencyScanningFilesystemSharedCache::CacheShard:: return *EntriesByFilename.insert({Filename, &Entry}).first->getValue(); } -/// Whitelist file extensions that should be minimized, treating no extension as -/// a source file that should be minimized. -/// -/// This is kinda hacky, it would be better if we knew what kind of file Clang -/// was expecting instead. -static bool shouldScanForDirectivesBasedOnExtension(StringRef Filename) { - StringRef Ext = llvm::sys::path::extension(Filename); - if (Ext.empty()) - return true; // C++ standard library - return llvm::StringSwitch(Ext) - .CasesLower(".c", ".cc", ".cpp", ".c++", ".cxx", true) - .CasesLower(".h", ".hh", ".hpp", ".h++", ".hxx", true) - .CasesLower(".m", ".mm", true) - .CasesLower(".i", ".ii", ".mi", ".mmi", true) - .CasesLower(".def", ".inc", true) - .Default(false); -} - static bool shouldCacheStatFailures(StringRef Filename) { StringRef Ext = llvm::sys::path::extension(Filename); if (Ext.empty()) return false; // This may be the module cache directory. - // Only cache stat failures on files that are not expected to change during - // the build. - StringRef FName = llvm::sys::path::filename(Filename); - if (FName == "module.modulemap" || FName == "module.map") - return true; - return shouldScanForDirectivesBasedOnExtension(Filename); + return true; } DependencyScanningWorkerFilesystem::DependencyScanningWorkerFilesystem( @@ -201,11 +179,6 @@ DependencyScanningWorkerFilesystem::DependencyScanningWorkerFilesystem( updateWorkingDirForCacheLookup(); } -bool DependencyScanningWorkerFilesystem::shouldScanForDirectives( - StringRef Filename) { - return shouldScanForDirectivesBasedOnExtension(Filename); -} - const CachedFileSystemEntry & DependencyScanningWorkerFilesystem::getOrEmplaceSharedEntryForUID( TentativeEntry TEntry) { @@ -259,7 +232,7 @@ DependencyScanningWorkerFilesystem::computeAndStoreResult( llvm::ErrorOr DependencyScanningWorkerFilesystem::getOrCreateFileSystemEntry( - StringRef OriginalFilename, bool DisableDirectivesScanning) { + StringRef OriginalFilename) { StringRef FilenameForLookup; SmallString<256> PathBuf; if (llvm::sys::path::is_absolute_gnu(OriginalFilename)) { @@ -276,15 +249,11 @@ DependencyScanningWorkerFilesystem::getOrCreateFileSystemEntry( assert(llvm::sys::path::is_absolute_gnu(FilenameForLookup)); if (const auto *Entry = findEntryByFilenameWithWriteThrough(FilenameForLookup)) - return scanForDirectivesIfNecessary(*Entry, OriginalFilename, - DisableDirectivesScanning) - .unwrapError(); + return EntryRef(OriginalFilename, *Entry).unwrapError(); auto MaybeEntry = computeAndStoreResult(OriginalFilename, FilenameForLookup); if (!MaybeEntry) return MaybeEntry.getError(); - return scanForDirectivesIfNecessary(*MaybeEntry, OriginalFilename, - DisableDirectivesScanning) - .unwrapError(); + return EntryRef(OriginalFilename, *MaybeEntry).unwrapError(); } llvm::ErrorOr diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp index 76f3d950a13b..33b43417a661 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp @@ -372,7 +372,8 @@ public: -> std::optional> { if (llvm::ErrorOr Entry = LocalDepFS->getOrCreateFileSystemEntry(File.getName())) - return Entry->getDirectiveTokens(); + if (LocalDepFS->ensureDirectiveTokensArePopulated(*Entry)) + return Entry->getDirectiveTokens(); return std::nullopt; }; } diff --git a/clang/test/ClangScanDeps/modules-extension.c b/clang/test/ClangScanDeps/modules-extension.c new file mode 100644 index 000000000000..0f27f608440f --- /dev/null +++ b/clang/test/ClangScanDeps/modules-extension.c @@ -0,0 +1,33 @@ +// RUN: rm -rf %t +// RUN: split-file %s %t + +// This test checks that source files with uncommon extensions still undergo +// dependency directives scan. If header.pch would not and b.h would, the scan +// would fail when parsing `void function(B)` and not knowing the symbol B. + +//--- module.modulemap +module __PCH { header "header.pch" } +module B { header "b.h" } + +//--- header.pch +#include "b.h" +void function(B); + +//--- b.h +typedef int B; + +//--- tu.c +int main() { + function(0); + return 0; +} + +//--- cdb.json.in +[{ + "directory": "DIR", + "file": "DIR/tu.c", + "command": "clang -c DIR/tu.c -fmodules -fmodules-cache-path=DIR/cache -fimplicit-module-maps -include DIR/header.pch" +}] + +// RUN: sed -e "s|DIR|%/t|g" %t/cdb.json.in > %t/cdb.json +// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full > %t/deps.json -- GitLab From af63c6e5d08fcaeacaeee68aa0a1cda71d9a7549 Mon Sep 17 00:00:00 2001 From: Philipp Tomsich Date: Sat, 23 Mar 2024 00:29:58 +0100 Subject: [PATCH 123/306] [AArch64] Adjust ROBsize for Ampere1/Ampere1A (NFC) (#86330) To align more closely with common usage, we now use the size of the reorder-buffer for MicroOpBufferSize instead of the entries of the global micro-op scheduler. --- llvm/lib/Target/AArch64/AArch64SchedAmpere1.td | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/Target/AArch64/AArch64SchedAmpere1.td b/llvm/lib/Target/AArch64/AArch64SchedAmpere1.td index cf9f50c2784b..269f4ec5e5fb 100644 --- a/llvm/lib/Target/AArch64/AArch64SchedAmpere1.td +++ b/llvm/lib/Target/AArch64/AArch64SchedAmpere1.td @@ -18,7 +18,7 @@ def Ampere1Model : SchedMachineModel { let IssueWidth = 4; // 4-way decode and dispatch - let MicroOpBufferSize = 174; // micro-op re-order buffer size + let MicroOpBufferSize = 192; // re-order buffer size let LoadLatency = 4; // Optimistic load latency let MispredictPenalty = 10; // Branch mispredict penalty let LoopMicroOpBufferSize = 32; // Instruction queue size -- GitLab From b621269d4a4c08269b1b2d46f277d1918d3dab62 Mon Sep 17 00:00:00 2001 From: Philipp Tomsich Date: Sat, 23 Mar 2024 00:30:12 +0100 Subject: [PATCH 124/306] [AArch64] Adjust ROBsize for Ampere1B (NFC) (#86331) To align more closely with common usage, we now use the size of the reorder-buffer for MicroOpBufferSize instead of the entries of the global micro-op scheduler. --- llvm/lib/Target/AArch64/AArch64SchedAmpere1B.td | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/Target/AArch64/AArch64SchedAmpere1B.td b/llvm/lib/Target/AArch64/AArch64SchedAmpere1B.td index 9c4f000cf351..67f8593f1577 100644 --- a/llvm/lib/Target/AArch64/AArch64SchedAmpere1B.td +++ b/llvm/lib/Target/AArch64/AArch64SchedAmpere1B.td @@ -18,7 +18,7 @@ def Ampere1BModel : SchedMachineModel { let IssueWidth = 12; // Maximum micro-ops dispatch rate. - let MicroOpBufferSize = 192; // micro-op re-order buffer size + let MicroOpBufferSize = 208; // micro-op re-order buffer size let LoadLatency = 3; // Optimistic load latency let MispredictPenalty = 10; // Branch mispredict penalty let LoopMicroOpBufferSize = 32; // Instruction queue size -- GitLab From c3747883a0bf34d271bc89dbfc60590adf75d999 Mon Sep 17 00:00:00 2001 From: Xiaoyang Liu Date: Fri, 22 Mar 2024 16:32:02 -0700 Subject: [PATCH 125/306] [libc++][ranges] use `static operator()` for C++23 ranges (#86052) ## Abstract This pull request converts the `operator()` of all CPOs and niebloids related to C++23 ranges to `static`. ## Motivation In `libc++`, CPOs and niebloids are implemented as function objects. Currently, the `operator()` for such a function object is a `const`-qualified member function. This means that even if the function object is has no data members, an extra register is used to pass in the `this` pointer when calling `operator()`, unless the compiler can inline the function call. Declaraing `operator()` as `static` would optimize away the unnecessary `this` pointer passing for stateless function objects, since there is no object instance state that needs to be accessed. ## Reference - [P1169R4: static `operator()`](https://wg21.link/P1169R4) --- libcxx/include/__algorithm/ranges_ends_with.h | 6 +++--- libcxx/include/__algorithm/ranges_starts_with.h | 8 ++++---- libcxx/include/__ranges/as_rvalue_view.h | 16 ++++++++-------- libcxx/include/__ranges/repeat_view.h | 5 ++--- libcxx/include/__ranges/to.h | 4 ++-- libcxx/include/__ranges/zip_view.h | 8 ++++---- 6 files changed, 23 insertions(+), 24 deletions(-) diff --git a/libcxx/include/__algorithm/ranges_ends_with.h b/libcxx/include/__algorithm/ranges_ends_with.h index c2a3cae9f3b1..bb01918326b8 100644 --- a/libcxx/include/__algorithm/ranges_ends_with.h +++ b/libcxx/include/__algorithm/ranges_ends_with.h @@ -39,7 +39,7 @@ namespace ranges { namespace __ends_with { struct __fn { template - static _LIBCPP_HIDE_FROM_ABI constexpr bool __ends_with_fn_impl_bidirectional( + _LIBCPP_HIDE_FROM_ABI static constexpr bool __ends_with_fn_impl_bidirectional( _Iter1 __first1, _Sent1 __last1, _Iter2 __first2, @@ -56,7 +56,7 @@ struct __fn { } template - static _LIBCPP_HIDE_FROM_ABI constexpr bool __ends_with_fn_impl( + _LIBCPP_HIDE_FROM_ABI static constexpr bool __ends_with_fn_impl( _Iter1 __first1, _Sent1 __last1, _Iter2 __first2, @@ -65,7 +65,7 @@ struct __fn { _Proj1& __proj1, _Proj2& __proj2) { if constexpr (std::bidirectional_iterator<_Sent1> && std::bidirectional_iterator<_Sent2> && - (!std::random_access_iterator<_Sent1>)&&(!std::random_access_iterator<_Sent2>)) { + (!std::random_access_iterator<_Sent1>) && (!std::random_access_iterator<_Sent2>)) { return __ends_with_fn_impl_bidirectional(__first1, __last1, __first2, __last2, __pred, __proj1, __proj2); } else { diff --git a/libcxx/include/__algorithm/ranges_starts_with.h b/libcxx/include/__algorithm/ranges_starts_with.h index 90e184aa9bcc..7ba8af13a8d1 100644 --- a/libcxx/include/__algorithm/ranges_starts_with.h +++ b/libcxx/include/__algorithm/ranges_starts_with.h @@ -42,14 +42,14 @@ struct __fn { class _Proj1 = identity, class _Proj2 = identity> requires indirectly_comparable<_Iter1, _Iter2, _Pred, _Proj1, _Proj2> - _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr bool operator()( + _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI static constexpr bool operator()( _Iter1 __first1, _Sent1 __last1, _Iter2 __first2, _Sent2 __last2, _Pred __pred = {}, _Proj1 __proj1 = {}, - _Proj2 __proj2 = {}) const { + _Proj2 __proj2 = {}) { return __mismatch::__fn::__go( std::move(__first1), std::move(__last1), @@ -67,8 +67,8 @@ struct __fn { class _Proj1 = identity, class _Proj2 = identity> requires indirectly_comparable, iterator_t<_Range2>, _Pred, _Proj1, _Proj2> - _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr bool operator()( - _Range1&& __range1, _Range2&& __range2, _Pred __pred = {}, _Proj1 __proj1 = {}, _Proj2 __proj2 = {}) const { + _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI static constexpr bool + operator()(_Range1&& __range1, _Range2&& __range2, _Pred __pred = {}, _Proj1 __proj1 = {}, _Proj2 __proj2 = {}) { return __mismatch::__fn::__go( ranges::begin(__range1), ranges::end(__range1), diff --git a/libcxx/include/__ranges/as_rvalue_view.h b/libcxx/include/__ranges/as_rvalue_view.h index 295aa94ed9fe..2fc272e798d6 100644 --- a/libcxx/include/__ranges/as_rvalue_view.h +++ b/libcxx/include/__ranges/as_rvalue_view.h @@ -111,18 +111,18 @@ namespace views { namespace __as_rvalue { struct __fn : __range_adaptor_closure<__fn> { template - _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Range&& __range) const - noexcept(noexcept(/**/ as_rvalue_view(std::forward<_Range>(__range)))) - -> decltype(/*--*/ as_rvalue_view(std::forward<_Range>(__range))) { - return /*-------------*/ as_rvalue_view(std::forward<_Range>(__range)); + _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI static constexpr auto + operator()(_Range&& __range) noexcept(noexcept(as_rvalue_view(std::forward<_Range>(__range)))) + -> decltype(/*--------------------------*/ as_rvalue_view(std::forward<_Range>(__range))) { + return /*---------------------------------*/ as_rvalue_view(std::forward<_Range>(__range)); } template requires same_as, range_reference_t<_Range>> - _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Range&& __range) const - noexcept(noexcept(/**/ views::all(std::forward<_Range>(__range)))) - -> decltype(/*--*/ views::all(std::forward<_Range>(__range))) { - return /*-------------*/ views::all(std::forward<_Range>(__range)); + _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI static constexpr auto + operator()(_Range&& __range) noexcept(noexcept(views::all(std::forward<_Range>(__range)))) + -> decltype(/*--------------------------*/ views::all(std::forward<_Range>(__range))) { + return /*---------------------------------*/ views::all(std::forward<_Range>(__range)); } }; } // namespace __as_rvalue diff --git a/libcxx/include/__ranges/repeat_view.h b/libcxx/include/__ranges/repeat_view.h index 620a26454972..5caea757a393 100644 --- a/libcxx/include/__ranges/repeat_view.h +++ b/libcxx/include/__ranges/repeat_view.h @@ -229,14 +229,13 @@ namespace views { namespace __repeat { struct __fn { template - _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __value) const + _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI static constexpr auto operator()(_Tp&& __value) noexcept(noexcept(ranges::repeat_view(std::forward<_Tp>(__value)))) -> decltype( ranges::repeat_view(std::forward<_Tp>(__value))) { return ranges::repeat_view(std::forward<_Tp>(__value)); } - template - _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __value, _Bound&& __bound_sentinel) const + _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI static constexpr auto operator()(_Tp&& __value, _Bound&& __bound_sentinel) noexcept(noexcept(ranges::repeat_view(std::forward<_Tp>(__value), std::forward<_Bound>(__bound_sentinel)))) -> decltype( ranges::repeat_view(std::forward<_Tp>(__value), std::forward<_Bound>(__bound_sentinel))) { return ranges::repeat_view(std::forward<_Tp>(__value), std::forward<_Bound>(__bound_sentinel)); } diff --git a/libcxx/include/__ranges/to.h b/libcxx/include/__ranges/to.h index cf162100ee46..67818c521b15 100644 --- a/libcxx/include/__ranges/to.h +++ b/libcxx/include/__ranges/to.h @@ -207,7 +207,7 @@ _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr auto to(_Args&&... __args) static_assert( !is_volatile_v<_Container>, "The target container cannot be volatile-qualified, please remove the volatile"); - auto __to_func = [](_Range&& __range, _Tail&&... __tail) + auto __to_func = [](_Range&& __range, _Tail&&... __tail) static requires requires { // /**/ ranges::to<_Container>(std::forward<_Range>(__range), std::forward<_Tail>(__tail)...); } @@ -223,7 +223,7 @@ _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr auto to(_Args&&... __args) // clang-format off auto __to_func = []::type> - (_Range&& __range, _Tail&& ... __tail) + (_Range&& __range, _Tail&& ... __tail) static requires requires { // /**/ ranges::to<_DeducedExpr>(std::forward<_Range>(__range), std::forward<_Tail>(__tail)...); } diff --git a/libcxx/include/__ranges/zip_view.h b/libcxx/include/__ranges/zip_view.h index ce00a4e53a48..d3665a149a7c 100644 --- a/libcxx/include/__ranges/zip_view.h +++ b/libcxx/include/__ranges/zip_view.h @@ -489,12 +489,12 @@ namespace views { namespace __zip { struct __fn { - _LIBCPP_HIDE_FROM_ABI constexpr auto operator()() const noexcept { return empty_view>{}; } + _LIBCPP_HIDE_FROM_ABI static constexpr auto operator()() noexcept { return empty_view>{}; } template - _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Ranges&&... __rs) const - noexcept(noexcept(zip_view...>(std::forward<_Ranges>(__rs)...))) - -> decltype(zip_view...>(std::forward<_Ranges>(__rs)...)) { + _LIBCPP_HIDE_FROM_ABI static constexpr auto + operator()(_Ranges&&... __rs) noexcept(noexcept(zip_view...>(std::forward<_Ranges>(__rs)...))) + -> decltype(zip_view...>(std::forward<_Ranges>(__rs)...)) { return zip_view...>(std::forward<_Ranges>(__rs)...); } }; -- GitLab From 20e0bacd0560382a31ad0d4ecc7472bd4a99c659 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Fri, 22 Mar 2024 19:08:47 -0500 Subject: [PATCH 126/306] [Libomptarget][Fix] Remove duplicate version script for host builds Summary: This causes an error on some linkers and was mistakenly kept in. --- openmp/libomptarget/plugins-nextgen/host/CMakeLists.txt | 5 ----- 1 file changed, 5 deletions(-) diff --git a/openmp/libomptarget/plugins-nextgen/host/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/host/CMakeLists.txt index d30680e10431..ccbf7d033fd6 100644 --- a/openmp/libomptarget/plugins-nextgen/host/CMakeLists.txt +++ b/openmp/libomptarget/plugins-nextgen/host/CMakeLists.txt @@ -31,11 +31,6 @@ else() target_include_directories(omptarget.rtl.${machine} PRIVATE dynamic_ffi) endif() -if(LIBOMP_HAVE_VERSION_SCRIPT_FLAG) - target_link_libraries(omptarget.rtl.${machine} PRIVATE - "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/../exports") -endif() - # Install plugin under the lib destination folder. install(TARGETS omptarget.rtl.${machine} LIBRARY DESTINATION "${OPENMP_INSTALL_LIBDIR}") -- GitLab From dc43aa439ecee90a9f51c6c1e46df8be61e0457c Mon Sep 17 00:00:00 2001 From: Charlie Barto Date: Fri, 22 Mar 2024 17:38:34 -0700 Subject: [PATCH 127/306] [asan][windows] When compiling with clang-cl or MSVC pass /Zl (that's a little L) (#85874) /Zl is the equivalent of -nodefaultlibs. The idea here is to make sure that the asan runtime doesn't have any defaultlibs directives, which makes it easier to mix an asan runtime built with the dynamic CRT with an application built with the static CRT (or vise-versa). This is part of the overall effort to remove the static asan runtime on windows entirely: https://github.com/llvm/llvm-project/pull/81677 Co-authored-by: Amy Wishnousky --- compiler-rt/lib/asan/CMakeLists.txt | 3 +++ compiler-rt/lib/sanitizer_common/CMakeLists.txt | 2 ++ compiler-rt/lib/ubsan/CMakeLists.txt | 1 + 3 files changed, 6 insertions(+) diff --git a/compiler-rt/lib/asan/CMakeLists.txt b/compiler-rt/lib/asan/CMakeLists.txt index f83ae82d4293..601750f72175 100644 --- a/compiler-rt/lib/asan/CMakeLists.txt +++ b/compiler-rt/lib/asan/CMakeLists.txt @@ -85,6 +85,9 @@ SET(ASAN_HEADERS include_directories(..) set(ASAN_CFLAGS ${SANITIZER_COMMON_CFLAGS}) + +append_list_if(MSVC /Zl ASAN_CFLAGS) + set(ASAN_COMMON_DEFINITIONS ${COMPILER_RT_ASAN_SHADOW_SCALE_DEFINITION}) append_rtti_flag(OFF ASAN_CFLAGS) diff --git a/compiler-rt/lib/sanitizer_common/CMakeLists.txt b/compiler-rt/lib/sanitizer_common/CMakeLists.txt index f762524c333a..f2b4ac72ae15 100644 --- a/compiler-rt/lib/sanitizer_common/CMakeLists.txt +++ b/compiler-rt/lib/sanitizer_common/CMakeLists.txt @@ -218,6 +218,8 @@ include_directories(..) set(SANITIZER_COMMON_DEFINITIONS HAVE_RPC_XDR_H=${HAVE_RPC_XDR_H}) +# note: L not I, this is nodefaultlibs for msvc +append_list_if(MSVC /Zl SANITIZER_COMMON_CFLAGS) set(SANITIZER_CFLAGS ${SANITIZER_COMMON_CFLAGS}) # Too many existing bugs, needs cleanup. diff --git a/compiler-rt/lib/ubsan/CMakeLists.txt b/compiler-rt/lib/ubsan/CMakeLists.txt index 3f1e12ed9ac6..db0b33f1276e 100644 --- a/compiler-rt/lib/ubsan/CMakeLists.txt +++ b/compiler-rt/lib/ubsan/CMakeLists.txt @@ -41,6 +41,7 @@ set(UBSAN_HEADERS include_directories(..) set(UBSAN_CFLAGS ${SANITIZER_COMMON_CFLAGS}) +append_list_if(MSVC /Zl UBSAN_CFLAGS) append_rtti_flag(OFF UBSAN_CFLAGS) append_list_if(SANITIZER_CAN_USE_CXXABI -DUBSAN_CAN_USE_CXXABI UBSAN_CFLAGS) -- GitLab From b723c57f8fdb12ca8eaa6c5ee2afb820aaeb56c5 Mon Sep 17 00:00:00 2001 From: Corentin Jabot Date: Sat, 23 Mar 2024 09:28:33 +0900 Subject: [PATCH 128/306] [Clang] Update the C++ page with papers approved in Tokyo --- clang/www/cxx_status.html | 42 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/clang/www/cxx_status.html b/clang/www/cxx_status.html index 1e36b90356c3..c1d95dadbb27 100755 --- a/clang/www/cxx_status.html +++ b/clang/www/cxx_status.html @@ -163,7 +163,47 @@ C++23, informally referred to as C++26.

P2864R2 Clang 18 - + + + Disallow Binding a Returned Glvalue to a Temporary + P2748R5 + No + + + Clarifying rules for brace elision in aggregate initialization + P3106R1 (DR) + No + + + Attributes for Structured Bindings + P0609R3 + No + + + Module Declarations Shouldn’t be Macros + P3034R1 (DR) + No + + + Trivial infinite loops are not Undefined Behavior + P2809R3 (DR) + No + + + Erroneous behaviour for uninitialized reads + P2795R5 + No + + + = delete("should have a reason"); + P2573R2 + No + + + Variadic friends + P2893R3 + No + -- GitLab From 2f6b1b4b30e3a719b1744baa4cd1ece504998c6e Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Fri, 22 Mar 2024 18:10:21 -0700 Subject: [PATCH 129/306] [ORC] Add default visibility to required JIT functions (#86322) If you build LLVM with `-DCMAKE_CXX_VISIBILITY_PRESET=hidden` to help reduce binary size, these symbols end up becoming local, and getting stripped. This forces default visibility to override the global setting in that case. Relevant: https://github.com/llvm/llvm-project/issues/62815#issuecomment-1560078260 --- llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.cpp b/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.cpp index 8a4145a6b02a..7529d9cef67e 100644 --- a/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.cpp +++ b/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.cpp @@ -26,11 +26,13 @@ extern "C" { // We put information about the JITed function in this global, which the // debugger reads. Make sure to specify the version statically, because the // debugger checks the version before we can set it during runtime. +LLVM_ATTRIBUTE_VISIBILITY_DEFAULT struct jit_descriptor __jit_debug_descriptor = {JitDescriptorVersion, 0, nullptr, nullptr}; // Debuggers that implement the GDB JIT interface put a special breakpoint in // this function. +LLVM_ATTRIBUTE_VISIBILITY_DEFAULT LLVM_ATTRIBUTE_NOINLINE void __jit_debug_register_code() { // The noinline and the asm prevent calls to this function from being // optimized out. -- GitLab From 84e20cb0cbc64bb2d8299b30da3ce8e6421c203e Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Fri, 22 Mar 2024 16:56:48 -0700 Subject: [PATCH 130/306] [TableGen] Minor cleanup in CodeGenRegBank::getConcatSubRegIndex. NFC Mark a variable const. Capitalize a variable name. I'm going to add HwMode support to this code and wanted to clean it up a bit beforehand. --- llvm/utils/TableGen/CodeGenRegisters.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/llvm/utils/TableGen/CodeGenRegisters.cpp b/llvm/utils/TableGen/CodeGenRegisters.cpp index 40af0d3077b2..e851d4c16bf7 100644 --- a/llvm/utils/TableGen/CodeGenRegisters.cpp +++ b/llvm/utils/TableGen/CodeGenRegisters.cpp @@ -1400,11 +1400,11 @@ CodeGenSubRegIndex *CodeGenRegBank::getConcatSubRegIndex( // None exists, synthesize one. std::string Name = Parts.front()->getName(); // Determine whether all parts are contiguous. - bool isContinuous = true; + bool IsContinuous = true; unsigned Size = Parts.front()->Size; unsigned LastOffset = Parts.front()->Offset; unsigned LastSize = Parts.front()->Size; - unsigned UnknownSize = (uint16_t)-1; + const unsigned UnknownSize = (uint16_t)-1; for (unsigned i = 1, e = Parts.size(); i != e; ++i) { Name += '_'; Name += Parts[i]->getName(); @@ -1413,13 +1413,13 @@ CodeGenSubRegIndex *CodeGenRegBank::getConcatSubRegIndex( else Size += Parts[i]->Size; if (LastSize == UnknownSize || Parts[i]->Offset != (LastOffset + LastSize)) - isContinuous = false; + IsContinuous = false; LastOffset = Parts[i]->Offset; LastSize = Parts[i]->Size; } Idx = createSubRegIndex(Name, Parts.front()->getNamespace()); Idx->Size = Size; - Idx->Offset = isContinuous ? Parts.front()->Offset : -1; + Idx->Offset = IsContinuous ? Parts.front()->Offset : -1; Idx->ConcatenationOf.assign(Parts.begin(), Parts.end()); return Idx; } -- GitLab From 76fdb5902fbadbc08c6742156071431d8ad801ea Mon Sep 17 00:00:00 2001 From: paperchalice Date: Sat, 23 Mar 2024 10:37:53 +0800 Subject: [PATCH 131/306] [NewPM][DirectX] Add DirectXPassRegistry.def NFCI (#86242) Prepare migration for dag-isel --- .../Target/DirectX/DirectXPassRegistry.def | 29 +++++++++++++++++++ .../Target/DirectX/DirectXTargetMachine.cpp | 20 ++----------- 2 files changed, 31 insertions(+), 18 deletions(-) create mode 100644 llvm/lib/Target/DirectX/DirectXPassRegistry.def diff --git a/llvm/lib/Target/DirectX/DirectXPassRegistry.def b/llvm/lib/Target/DirectX/DirectXPassRegistry.def new file mode 100644 index 000000000000..bbf0c254bb69 --- /dev/null +++ b/llvm/lib/Target/DirectX/DirectXPassRegistry.def @@ -0,0 +1,29 @@ +//===- DirectXPassRegistry.def - Registry of DirectX passes -----*- C++--*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is used as the registry of passes that are part of the +// DirectX backend. +// +//===----------------------------------------------------------------------===// + +// NOTE: NO INCLUDE GUARD DESIRED! + +#ifndef MODULE_ANALYSIS +#define MODULE_ANALYSIS(NAME, CREATE_PASS) +#endif +MODULE_ANALYSIS("dx-shader-flags", dxil::ShaderFlagsAnalysis()) +MODULE_ANALYSIS("dxil-resource", DXILResourceAnalysis()) +#undef MODULE_ANALYSIS + +#ifndef MODULE_PASS +#define MODULE_PASS(NAME, CREATE_PASS) +#endif +// TODO: rename to print after NPM switch +MODULE_PASS("print-dx-shader-flags", DXILResourcePrinterPass(dbgs())) +MODULE_PASS("print-dxil-resource", DXILResourcePrinterPass(dbgs())) +#undef MODULE_PASS diff --git a/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp b/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp index 03c825b3977d..bebca0675522 100644 --- a/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp +++ b/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp @@ -104,24 +104,8 @@ DirectXTargetMachine::~DirectXTargetMachine() {} void DirectXTargetMachine::registerPassBuilderCallbacks( PassBuilder &PB, bool PopulateClassToPassNames) { - PB.registerPipelineParsingCallback( - [](StringRef PassName, ModulePassManager &PM, - ArrayRef) { - if (PassName == "print-dxil-resource") { - PM.addPass(DXILResourcePrinterPass(dbgs())); - return true; - } - if (PassName == "print-dx-shader-flags") { - PM.addPass(dxil::ShaderFlagsAnalysisPrinter(dbgs())); - return true; - } - return false; - }); - - PB.registerAnalysisRegistrationCallback([](ModuleAnalysisManager &MAM) { - MAM.registerPass([&] { return DXILResourceAnalysis(); }); - MAM.registerPass([&] { return dxil::ShaderFlagsAnalysis(); }); - }); +#define GET_PASS_REGISTRY "DirectXPassRegistry.def" +#include "llvm/Passes/TargetPassRegistry.inc" } bool DirectXTargetMachine::addPassesToEmitFile( -- GitLab From 7ac7d418ac2b16fd44789dcf48e2b5d73de3e715 Mon Sep 17 00:00:00 2001 From: paperchalice Date: Sat, 23 Mar 2024 11:20:18 +0800 Subject: [PATCH 132/306] [NewPM][NVPTX] Add NVPTXPassRegistry.def NFCI (#86246) Prepare for dag-isel migration. --- llvm/lib/Target/NVPTX/NVPTXPassRegistry.def | 40 +++++++++++++++++++ llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp | 41 +------------------- 2 files changed, 42 insertions(+), 39 deletions(-) create mode 100644 llvm/lib/Target/NVPTX/NVPTXPassRegistry.def diff --git a/llvm/lib/Target/NVPTX/NVPTXPassRegistry.def b/llvm/lib/Target/NVPTX/NVPTXPassRegistry.def new file mode 100644 index 000000000000..6ff15ab6f13c --- /dev/null +++ b/llvm/lib/Target/NVPTX/NVPTXPassRegistry.def @@ -0,0 +1,40 @@ +//===- NVPTXPassRegistry.def - Registry of NVPTX passes ---------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is used as the registry of passes that are part of the +// NVPTX backend. +// +//===----------------------------------------------------------------------===// + +// NOTE: NO INCLUDE GUARD DESIRED! + +#ifndef MODULE_PASS +#define MODULE_PASS(NAME, CREATE_PASS) +#endif +MODULE_PASS("generic-to-nvvm", GenericToNVVMPass()) +MODULE_PASS("nvptx-lower-ctor-dtor", NVPTXCtorDtorLoweringPass()) +#undef MODULE_PASS + +#ifndef FUNCTION_ANALYSIS +#define FUNCTION_ANALYSIS(NAME, CREATE_PASS) +#endif + +#ifndef FUNCTION_ALIAS_ANALYSIS +#define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS) \ + FUNCTION_ANALYSIS(NAME, CREATE_PASS) +#endif +FUNCTION_ALIAS_ANALYSIS("nvptx-aa", NVPTXAA()) +#undef FUNCTION_ALIAS_ANALYSIS +#undef FUNCTION_ANALYSIS + +#ifndef FUNCTION_PASS +#define FUNCTION_PASS(NAME, CREATE_PASS) +#endif +FUNCTION_PASS("nvvm-intr-range", NVVMIntrRangePass()) +FUNCTION_PASS("nvvm-reflect", NVVMReflectPass()) +#undef FUNCTION_PASS diff --git a/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp b/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp index 69d4596f7843..78f48652c992 100644 --- a/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp +++ b/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp @@ -227,45 +227,8 @@ void NVPTXTargetMachine::registerDefaultAliasAnalyses(AAManager &AAM) { void NVPTXTargetMachine::registerPassBuilderCallbacks( PassBuilder &PB, bool PopulateClassToPassNames) { - PB.registerPipelineParsingCallback( - [](StringRef PassName, FunctionPassManager &PM, - ArrayRef) { - if (PassName == "nvvm-reflect") { - PM.addPass(NVVMReflectPass()); - return true; - } - if (PassName == "nvvm-intr-range") { - PM.addPass(NVVMIntrRangePass()); - return true; - } - return false; - }); - - PB.registerAnalysisRegistrationCallback([](FunctionAnalysisManager &FAM) { - FAM.registerPass([&] { return NVPTXAA(); }); - }); - - PB.registerParseAACallback([](StringRef AAName, AAManager &AAM) { - if (AAName == "nvptx-aa") { - AAM.registerFunctionAnalysis(); - return true; - } - return false; - }); - - PB.registerPipelineParsingCallback( - [](StringRef PassName, ModulePassManager &PM, - ArrayRef) { - if (PassName == "nvptx-lower-ctor-dtor") { - PM.addPass(NVPTXCtorDtorLoweringPass()); - return true; - } - if (PassName == "generic-to-nvvm") { - PM.addPass(GenericToNVVMPass()); - return true; - } - return false; - }); +#define GET_PASS_REGISTRY "NVPTXPassRegistry.def" +#include "llvm/Passes/TargetPassRegistry.inc" PB.registerPipelineStartEPCallback( [this](ModulePassManager &PM, OptimizationLevel Level) { -- GitLab From 2aa5bae0c03f1f857d0ae2a881b223c4a521853f Mon Sep 17 00:00:00 2001 From: paperchalice Date: Sat, 23 Mar 2024 12:53:26 +0800 Subject: [PATCH 133/306] [NewPM][BPF] Add BPFPassRegistry.def NFCI (#86241) Prepare migration for dag-isel. --- llvm/lib/Target/BPF/BPFPassRegistry.def | 32 ++++++++++++++++++++++++ llvm/lib/Target/BPF/BPFTargetMachine.cpp | 25 ++++++------------ 2 files changed, 40 insertions(+), 17 deletions(-) create mode 100644 llvm/lib/Target/BPF/BPFPassRegistry.def diff --git a/llvm/lib/Target/BPF/BPFPassRegistry.def b/llvm/lib/Target/BPF/BPFPassRegistry.def new file mode 100644 index 000000000000..73a8ef2f95bf --- /dev/null +++ b/llvm/lib/Target/BPF/BPFPassRegistry.def @@ -0,0 +1,32 @@ +//===- BPFPassRegistry.def - Registry of BPF passes -------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is used as the registry of passes that are part of the +// BPF backend. +// +//===----------------------------------------------------------------------===// + +// NOTE: NO INCLUDE GUARD DESIRED! + +#ifndef FUNCTION_PASS +#define FUNCTION_PASS(NAME, CREATE_PASS) +#endif +FUNCTION_PASS("bpf-aspace-simplify", BPFASpaceCastSimplifyPass()) +FUNCTION_PASS("bpf-ir-peephole", BPFIRPeepholePass()) +#undef FUNCTION_PASS + +#ifndef FUNCTION_PASS_WITH_PARAMS +#define FUNCTION_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS) +#endif +FUNCTION_PASS_WITH_PARAMS( + "bpf-preserve-static-offset", "BPFPreserveStaticOffsetPass", + [=](bool AllowPartial) { + return BPFPreserveStaticOffsetPass(AllowPartial); + }, + parseBPFPreserveStaticOffsetOptions, "allow-partial") +#undef FUNCTION_PASS_WITH_PARAMS diff --git a/llvm/lib/Target/BPF/BPFTargetMachine.cpp b/llvm/lib/Target/BPF/BPFTargetMachine.cpp index 5f26bec2e390..a7bed69b0f2a 100644 --- a/llvm/lib/Target/BPF/BPFTargetMachine.cpp +++ b/llvm/lib/Target/BPF/BPFTargetMachine.cpp @@ -108,25 +108,16 @@ TargetPassConfig *BPFTargetMachine::createPassConfig(PassManagerBase &PM) { return new BPFPassConfig(*this, PM); } +static Expected parseBPFPreserveStaticOffsetOptions(StringRef Params) { + return PassBuilder::parseSinglePassOption(Params, "allow-partial", + "BPFPreserveStaticOffsetPass"); +} + void BPFTargetMachine::registerPassBuilderCallbacks( PassBuilder &PB, bool PopulateClassToPassNames) { - PB.registerPipelineParsingCallback( - [](StringRef PassName, FunctionPassManager &FPM, - ArrayRef) { - if (PassName == "bpf-ir-peephole") { - FPM.addPass(BPFIRPeepholePass()); - return true; - } - if (PassName == "bpf-preserve-static-offset") { - FPM.addPass(BPFPreserveStaticOffsetPass(false)); - return true; - } - if (PassName == "bpf-aspace-simplify") { - FPM.addPass(BPFASpaceCastSimplifyPass()); - return true; - } - return false; - }); +#define GET_PASS_REGISTRY "BPFPassRegistry.def" +#include "llvm/Passes/TargetPassRegistry.inc" + PB.registerPipelineStartEPCallback( [=](ModulePassManager &MPM, OptimizationLevel) { FunctionPassManager FPM; -- GitLab From 635ea257eca7c8e95c6ea30ca3816a0b5584ab37 Mon Sep 17 00:00:00 2001 From: paperchalice Date: Sat, 23 Mar 2024 13:06:58 +0800 Subject: [PATCH 134/306] [NewPM] Fix BPF build (#86379) Add Passes in dependency list --- llvm/lib/Target/BPF/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/lib/Target/BPF/CMakeLists.txt b/llvm/lib/Target/BPF/CMakeLists.txt index cb21ed03a86c..eade4cacb710 100644 --- a/llvm/lib/Target/BPF/CMakeLists.txt +++ b/llvm/lib/Target/BPF/CMakeLists.txt @@ -54,6 +54,7 @@ add_llvm_target(BPFCodeGen GlobalISel IPO MC + Passes Scalar SelectionDAG Support -- GitLab From 6c1932ffd82e733325180fe13ef46b24ff606eab Mon Sep 17 00:00:00 2001 From: Yingwei Zheng Date: Sat, 23 Mar 2024 14:57:35 +0800 Subject: [PATCH 135/306] [LLVM] Pass APInt by const reference. NFC. (#86278) This patch adjusts argument passing for `APInt` to improve the compile-time. Compile-time improvement: https://llvm-compile-time-tracker.com/compare.php?from=d1f182c895728d89c5c3d198b133e212a5d9d4a3&to=32d6611af69bf4e76373f9bc7d9649650f760e48&stat=instructions:u --- llvm/include/llvm/Analysis/MemoryBuiltins.h | 2 +- llvm/include/llvm/CodeGen/SelectionDAG.h | 2 +- llvm/include/llvm/MC/MCStreamer.h | 2 +- llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp | 2 +- llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 3 ++- llvm/lib/MC/MCStreamer.cpp | 2 +- 6 files changed, 7 insertions(+), 6 deletions(-) diff --git a/llvm/include/llvm/Analysis/MemoryBuiltins.h b/llvm/include/llvm/Analysis/MemoryBuiltins.h index 37ce1518f00c..da092866de65 100644 --- a/llvm/include/llvm/Analysis/MemoryBuiltins.h +++ b/llvm/include/llvm/Analysis/MemoryBuiltins.h @@ -217,7 +217,7 @@ struct SizeOffsetAPInt : public SizeOffsetType { SizeOffsetAPInt() = default; SizeOffsetAPInt(APInt Size, APInt Offset) : SizeOffsetType(Size, Offset) {} - static bool known(APInt V) { return V.getBitWidth() > 1; } + static bool known(const APInt &V) { return V.getBitWidth() > 1; } }; /// Evaluate the size and offset of an object pointed to by a Value* diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h index 4785e93d72d1..574c63552ce0 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAG.h +++ b/llvm/include/llvm/CodeGen/SelectionDAG.h @@ -883,7 +883,7 @@ public: /// Returns a vector of type ResVT whose elements contain the linear sequence /// <0, Step, Step * 2, Step * 3, ...> - SDValue getStepVector(const SDLoc &DL, EVT ResVT, APInt StepVal); + SDValue getStepVector(const SDLoc &DL, EVT ResVT, const APInt &StepVal); /// Returns a vector of type ResVT whose elements contain the linear sequence /// <0, 1, 2, 3, ...> diff --git a/llvm/include/llvm/MC/MCStreamer.h b/llvm/include/llvm/MC/MCStreamer.h index 671511ab4b88..69867620e1bf 100644 --- a/llvm/include/llvm/MC/MCStreamer.h +++ b/llvm/include/llvm/MC/MCStreamer.h @@ -740,7 +740,7 @@ public: /// Special case of EmitValue that avoids the client having /// to pass in a MCExpr for constant integers. virtual void emitIntValue(uint64_t Value, unsigned Size); - virtual void emitIntValue(APInt Value); + virtual void emitIntValue(const APInt &Value); /// Special case of EmitValue that avoids the client having to pass /// in a MCExpr for constant integers & prints in Hex format for certain diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp index 8d608f6ac5e4..1b25da8833e4 100644 --- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp @@ -7855,7 +7855,7 @@ LegalizerHelper::LegalizeResult LegalizerHelper::lowerBswap(MachineInstr &MI) { //{ (Src & Mask) >> N } | { (Src << N) & Mask } static MachineInstrBuilder SwapN(unsigned N, DstOp Dst, MachineIRBuilder &B, - MachineInstrBuilder Src, APInt Mask) { + MachineInstrBuilder Src, const APInt &Mask) { const LLT Ty = Dst.getLLTTy(*B.getMRI()); MachineInstrBuilder C_N = B.buildConstant(Ty, N); MachineInstrBuilder MaskLoNTo0 = B.buildConstant(Ty, Mask); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index cd6f083243d0..e2c07e7cb997 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -2031,7 +2031,8 @@ SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT) { return getStepVector(DL, ResVT, One); } -SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT, APInt StepVal) { +SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT, + const APInt &StepVal) { assert(ResVT.getScalarSizeInBits() == StepVal.getBitWidth()); if (ResVT.isScalableVector()) return getNode( diff --git a/llvm/lib/MC/MCStreamer.cpp b/llvm/lib/MC/MCStreamer.cpp index d0395770ae8b..176d55aa890b 100644 --- a/llvm/lib/MC/MCStreamer.cpp +++ b/llvm/lib/MC/MCStreamer.cpp @@ -141,7 +141,7 @@ void MCStreamer::emitIntValue(uint64_t Value, unsigned Size) { unsigned Index = IsLittleEndian ? 0 : 8 - Size; emitBytes(StringRef(reinterpret_cast(&Swapped) + Index, Size)); } -void MCStreamer::emitIntValue(APInt Value) { +void MCStreamer::emitIntValue(const APInt &Value) { if (Value.getNumWords() == 1) { emitIntValue(Value.getLimitedValue(), Value.getBitWidth() / 8); return; -- GitLab From 2f1f6b704d83f87be7ea885480caf1c86d8cfaee Mon Sep 17 00:00:00 2001 From: Yingwei Zheng Date: Sat, 23 Mar 2024 14:58:25 +0800 Subject: [PATCH 136/306] [LLVM] Use `std::move` for APInt. NFC. (#86257) This patch adjusts argument passing for `APInt` to improve the compile-time. Compile-time improvement: https://llvm-compile-time-tracker.com/compare.php?from=d1f182c895728d89c5c3d198b133e212a5d9d4a3&to=ba3e326def3a6e5cd6d72ff5a49c74fba18de1df&stat=instructions:u --- llvm/include/llvm/Analysis/InlineCost.h | 3 ++- llvm/include/llvm/Analysis/MemoryBuiltins.h | 6 ++++-- llvm/lib/Analysis/ConstantFolding.cpp | 2 +- llvm/lib/Analysis/InstructionSimplify.cpp | 6 ++++-- llvm/lib/IR/LLVMContextImpl.h | 2 +- llvm/lib/Transforms/Scalar/MergeICmps.cpp | 2 +- 6 files changed, 13 insertions(+), 8 deletions(-) diff --git a/llvm/include/llvm/Analysis/InlineCost.h b/llvm/include/llvm/Analysis/InlineCost.h index 3a760e0a85ce..c5978ce54fc1 100644 --- a/llvm/include/llvm/Analysis/InlineCost.h +++ b/llvm/include/llvm/Analysis/InlineCost.h @@ -65,7 +65,8 @@ const char MaxInlineStackSizeAttributeName[] = "inline-max-stacksize"; // The cost-benefit pair computed by cost-benefit analysis. class CostBenefitPair { public: - CostBenefitPair(APInt Cost, APInt Benefit) : Cost(Cost), Benefit(Benefit) {} + CostBenefitPair(APInt Cost, APInt Benefit) + : Cost(std::move(Cost)), Benefit(std::move(Benefit)) {} const APInt &getCost() const { return Cost; } diff --git a/llvm/include/llvm/Analysis/MemoryBuiltins.h b/llvm/include/llvm/Analysis/MemoryBuiltins.h index da092866de65..bb282a1b73d3 100644 --- a/llvm/include/llvm/Analysis/MemoryBuiltins.h +++ b/llvm/include/llvm/Analysis/MemoryBuiltins.h @@ -196,7 +196,8 @@ public: T Offset; SizeOffsetType() = default; - SizeOffsetType(T Size, T Offset) : Size(Size), Offset(Offset) {} + SizeOffsetType(T Size, T Offset) + : Size(std::move(Size)), Offset(std::move(Offset)) {} bool knownSize() const { return C::known(Size); } bool knownOffset() const { return C::known(Offset); } @@ -215,7 +216,8 @@ public: /// \p APInts. struct SizeOffsetAPInt : public SizeOffsetType { SizeOffsetAPInt() = default; - SizeOffsetAPInt(APInt Size, APInt Offset) : SizeOffsetType(Size, Offset) {} + SizeOffsetAPInt(APInt Size, APInt Offset) + : SizeOffsetType(std::move(Size), std::move(Offset)) {} static bool known(const APInt &V) { return V.getBitWidth() > 1; } }; diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp index 6139b5be85be..749374a3aa48 100644 --- a/llvm/lib/Analysis/ConstantFolding.cpp +++ b/llvm/lib/Analysis/ConstantFolding.cpp @@ -751,7 +751,7 @@ Constant *llvm::ConstantFoldLoadFromConstPtr(Constant *C, Type *Ty, Constant *llvm::ConstantFoldLoadFromConstPtr(Constant *C, Type *Ty, const DataLayout &DL) { APInt Offset(DL.getIndexTypeSizeInBits(C->getType()), 0); - return ConstantFoldLoadFromConstPtr(C, Ty, Offset, DL); + return ConstantFoldLoadFromConstPtr(C, Ty, std::move(Offset), DL); } Constant *llvm::ConstantFoldLoadFromUniformValue(Constant *C, Type *Ty, diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp index 7a37ae86c7f3..9ff3faff7990 100644 --- a/llvm/lib/Analysis/InstructionSimplify.cpp +++ b/llvm/lib/Analysis/InstructionSimplify.cpp @@ -6115,7 +6115,8 @@ static Value *simplifyRelativeLoad(Constant *Ptr, Constant *Offset, if (OffsetInt.srem(4) != 0) return nullptr; - Constant *Loaded = ConstantFoldLoadFromConstPtr(Ptr, Int32Ty, OffsetInt, DL); + Constant *Loaded = + ConstantFoldLoadFromConstPtr(Ptr, Int32Ty, std::move(OffsetInt), DL); if (!Loaded) return nullptr; @@ -6983,7 +6984,8 @@ Value *llvm::simplifyLoadInst(LoadInst *LI, Value *PtrOp, if (PtrOp == GV) { // Index size may have changed due to address space casts. Offset = Offset.sextOrTrunc(Q.DL.getIndexTypeSizeInBits(PtrOp->getType())); - return ConstantFoldLoadFromConstPtr(GV, LI->getType(), Offset, Q.DL); + return ConstantFoldLoadFromConstPtr(GV, LI->getType(), std::move(Offset), + Q.DL); } return nullptr; diff --git a/llvm/lib/IR/LLVMContextImpl.h b/llvm/lib/IR/LLVMContextImpl.h index 58e0f21244f7..7c67e191348e 100644 --- a/llvm/lib/IR/LLVMContextImpl.h +++ b/llvm/lib/IR/LLVMContextImpl.h @@ -441,7 +441,7 @@ template <> struct MDNodeKeyImpl { bool IsUnsigned; MDNodeKeyImpl(APInt Value, bool IsUnsigned, MDString *Name) - : Value(Value), Name(Name), IsUnsigned(IsUnsigned) {} + : Value(std::move(Value)), Name(Name), IsUnsigned(IsUnsigned) {} MDNodeKeyImpl(int64_t Value, bool IsUnsigned, MDString *Name) : Value(APInt(64, Value, !IsUnsigned)), Name(Name), IsUnsigned(IsUnsigned) {} diff --git a/llvm/lib/Transforms/Scalar/MergeICmps.cpp b/llvm/lib/Transforms/Scalar/MergeICmps.cpp index 1e0906717549..2bd13556c696 100644 --- a/llvm/lib/Transforms/Scalar/MergeICmps.cpp +++ b/llvm/lib/Transforms/Scalar/MergeICmps.cpp @@ -74,7 +74,7 @@ namespace { struct BCEAtom { BCEAtom() = default; BCEAtom(GetElementPtrInst *GEP, LoadInst *LoadI, int BaseId, APInt Offset) - : GEP(GEP), LoadI(LoadI), BaseId(BaseId), Offset(Offset) {} + : GEP(GEP), LoadI(LoadI), BaseId(BaseId), Offset(std::move(Offset)) {} BCEAtom(const BCEAtom &) = delete; BCEAtom &operator=(const BCEAtom &) = delete; -- GitLab From ef57977f2aa32661a09fa6538f47ddee0a004d11 Mon Sep 17 00:00:00 2001 From: paperchalice Date: Sat, 23 Mar 2024 15:02:27 +0800 Subject: [PATCH 137/306] [NewPM][Hexagon] Add HexagonPassRegistry.def (#86244) Prepare for dag-isel, also migrate some test case --- .../Target/Hexagon/HexagonPassRegistry.def | 21 +++++++++++++++++++ .../Target/Hexagon/HexagonTargetMachine.cpp | 3 +++ .../hexagon_vector_loop_carried_reuse.ll | 2 +- ...n_vector_loop_carried_reuse_commutative.ll | 2 +- ...agon_vector_loop_carried_reuse_constant.ll | 2 +- ...xagon_vector_loop_carried_reuse_invalid.ll | 2 +- .../CodeGen/Hexagon/hvx-loopidiom-memcpy.ll | 1 + .../Hexagon/loop-idiom/hexagon-memmove1.ll | 2 ++ .../Hexagon/loop-idiom/hexagon-memmove2.ll | 2 ++ llvm/test/CodeGen/Hexagon/loop-idiom/lcssa.ll | 2 +- .../Hexagon/loop-idiom/memmove-rt-check.ll | 1 + .../Hexagon/loop-idiom/nullptr-crash.ll | 1 + .../Hexagon/loop-idiom/pmpy-infinite-loop.ll | 1 + .../Hexagon/loop-idiom/pmpy-long-loop.ll | 1 + .../Hexagon/loop-idiom/pmpy-shiftconv-fail.ll | 1 + llvm/test/CodeGen/Hexagon/loop-idiom/pmpy.ll | 2 ++ 16 files changed, 41 insertions(+), 5 deletions(-) create mode 100644 llvm/lib/Target/Hexagon/HexagonPassRegistry.def diff --git a/llvm/lib/Target/Hexagon/HexagonPassRegistry.def b/llvm/lib/Target/Hexagon/HexagonPassRegistry.def new file mode 100644 index 000000000000..4f58ae6193e0 --- /dev/null +++ b/llvm/lib/Target/Hexagon/HexagonPassRegistry.def @@ -0,0 +1,21 @@ +//===- HexagonPassRegistry.def - Registry of Hexagon passes -----*- C++--*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is used as the registry of passes that are part of the +// Hexagon backend. +// +//===----------------------------------------------------------------------===// + +// NOTE: NO INCLUDE GUARD DESIRED! + +#ifndef LOOP_PASS +#define LOOP_PASS(NAME, CREATE_PASS) +#endif +LOOP_PASS("hexagon-loop-idiom", HexagonLoopIdiomRecognitionPass()) +LOOP_PASS("hexagon-vlcr", HexagonVectorLoopCarriedReusePass()) +#undef LOOP_PASS diff --git a/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp b/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp index 3c346c334d6d..e64d7e52a9aa 100644 --- a/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp +++ b/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp @@ -299,6 +299,9 @@ HexagonTargetMachine::getSubtargetImpl(const Function &F) const { void HexagonTargetMachine::registerPassBuilderCallbacks( PassBuilder &PB, bool PopulateClassToPassNames) { +#define GET_PASS_REGISTRY "HexagonPassRegistry.def" +#include "llvm/Passes/TargetPassRegistry.inc" + PB.registerLateLoopOptimizationsEPCallback( [=](LoopPassManager &LPM, OptimizationLevel Level) { LPM.addPass(HexagonLoopIdiomRecognitionPass()); diff --git a/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse.ll b/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse.ll index 0771fda02cfb..7ccee1689185 100644 --- a/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse.ll +++ b/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse.ll @@ -1,4 +1,4 @@ -; RUN: opt < %s -hexagon-vlcr | opt -passes=adce -S | FileCheck %s +; RUN: opt -mtriple=hexagon-- -passes='loop(hexagon-vlcr),adce' -S %s | FileCheck %s ; CHECK: %.hexagon.vlcr = tail call <32 x i32> @llvm.hexagon.V6.vmaxub.128B ; ModuleID = 'hexagon_vector_loop_carried_reuse.c' diff --git a/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_commutative.ll b/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_commutative.ll index 25afb9f1a137..532f7fd06793 100644 --- a/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_commutative.ll +++ b/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_commutative.ll @@ -1,4 +1,4 @@ -; RUN: opt < %s -march=hexagon -hexagon-vlcr | opt -passes=adce -S | FileCheck %s +; RUN: opt -mtriple hexagon-- -passes='loop(hexagon-vlcr),adce' -S %s | FileCheck %s ; CHECK: %v32.hexagon.vlcr = tail call <32 x i32> @llvm.hexagon.V6.vmaxub.128B diff --git a/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_constant.ll b/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_constant.ll index 53973423732c..ecfcf53d9133 100644 --- a/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_constant.ll +++ b/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_constant.ll @@ -1,4 +1,4 @@ -; RUN: opt < %s -hexagon-vlcr | opt -passes=adce -S | FileCheck %s +; RUN: opt -mtriple=hexagon-- -passes='loop(hexagon-vlcr),adce' -S %s | FileCheck %s ; CHECK-NOT: %.hexagon.vlcr ; ModuleID = 'hexagon_vector_loop_carried_reuse.c' diff --git a/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_invalid.ll b/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_invalid.ll index b440dba66f67..9872faeb3da0 100644 --- a/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_invalid.ll +++ b/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_invalid.ll @@ -1,4 +1,4 @@ -; RUN: opt -hexagon-vlcr < %s -S | FileCheck %s +; RUN: opt -mtriple=hexagon-- -passes=hexagon-vlcr -S %s | FileCheck %s ; Test that reuse doesn't occur due to two shufflevectors with different masks. diff --git a/llvm/test/CodeGen/Hexagon/hvx-loopidiom-memcpy.ll b/llvm/test/CodeGen/Hexagon/hvx-loopidiom-memcpy.ll index ab7bf1b4b0e8..c53e57800edd 100644 --- a/llvm/test/CodeGen/Hexagon/hvx-loopidiom-memcpy.ll +++ b/llvm/test/CodeGen/Hexagon/hvx-loopidiom-memcpy.ll @@ -1,4 +1,5 @@ ; RUN: opt -march=hexagon -hexagon-loop-idiom -S < %s | FileCheck %s +; RUN: opt -mtriple=hexagon-- -p hexagon-loop-idiom -disable-memcpy-idiom -S < %s | FileCheck %s ; Make sure we don't convert load/store loops into memcpy if the access type ; is a vector. Using vector instructions is generally better in such cases. diff --git a/llvm/test/CodeGen/Hexagon/loop-idiom/hexagon-memmove1.ll b/llvm/test/CodeGen/Hexagon/loop-idiom/hexagon-memmove1.ll index c7110263c658..5ace9e6ee486 100644 --- a/llvm/test/CodeGen/Hexagon/loop-idiom/hexagon-memmove1.ll +++ b/llvm/test/CodeGen/Hexagon/loop-idiom/hexagon-memmove1.ll @@ -1,6 +1,8 @@ ; Check for recognizing the "memmove" idiom. ; RUN: opt -hexagon-loop-idiom -S -mtriple hexagon-unknown-elf < %s \ ; RUN: | FileCheck %s +; RUN: opt -p hexagon-loop-idiom -S -mtriple hexagon-unknown-elf < %s \ +; RUN: | FileCheck %s ; CHECK: call void @llvm.memmove ; Function Attrs: norecurse nounwind diff --git a/llvm/test/CodeGen/Hexagon/loop-idiom/hexagon-memmove2.ll b/llvm/test/CodeGen/Hexagon/loop-idiom/hexagon-memmove2.ll index 234e4f56b5d8..ed56a332f657 100644 --- a/llvm/test/CodeGen/Hexagon/loop-idiom/hexagon-memmove2.ll +++ b/llvm/test/CodeGen/Hexagon/loop-idiom/hexagon-memmove2.ll @@ -1,5 +1,7 @@ ; RUN: opt -hexagon-loop-idiom -S -mtriple hexagon-unknown-elf < %s \ ; RUN: | FileCheck %s +; RUN: opt -p hexagon-loop-idiom -S -mtriple hexagon-unknown-elf < %s \ +; RUN: | FileCheck %s define void @PR14241(ptr %s, i64 %size) #0 { ; Ensure that we don't form a memcpy for strided loops. Briefly, when we taught diff --git a/llvm/test/CodeGen/Hexagon/loop-idiom/lcssa.ll b/llvm/test/CodeGen/Hexagon/loop-idiom/lcssa.ll index 140c676175ea..e5bcc2b9aebf 100644 --- a/llvm/test/CodeGen/Hexagon/loop-idiom/lcssa.ll +++ b/llvm/test/CodeGen/Hexagon/loop-idiom/lcssa.ll @@ -1,4 +1,4 @@ -; RUN: opt -S -hexagon-loop-idiom < %s | opt -S -passes='loop(loop-deletion),gvn' +; RUN: opt -mtriple hexagon-- -S -passes='loop(hexagon-loop-idiom,loop-deletion),gvn' ; REQUIRES: asserts ; This tests that the HexagonLoopIdiom pass does not mark LCSSA information diff --git a/llvm/test/CodeGen/Hexagon/loop-idiom/memmove-rt-check.ll b/llvm/test/CodeGen/Hexagon/loop-idiom/memmove-rt-check.ll index 7a7d1d9b1a86..78f0c9e36b55 100644 --- a/llvm/test/CodeGen/Hexagon/loop-idiom/memmove-rt-check.ll +++ b/llvm/test/CodeGen/Hexagon/loop-idiom/memmove-rt-check.ll @@ -1,4 +1,5 @@ ; RUN: opt -hexagon-loop-idiom -S < %s | FileCheck %s +; RUN: opt -p hexagon-loop-idiom -S < %s | FileCheck %s ; Make sure that we generate correct runtime checks. diff --git a/llvm/test/CodeGen/Hexagon/loop-idiom/nullptr-crash.ll b/llvm/test/CodeGen/Hexagon/loop-idiom/nullptr-crash.ll index 37e1bb6eb7df..ce02b62911c0 100644 --- a/llvm/test/CodeGen/Hexagon/loop-idiom/nullptr-crash.ll +++ b/llvm/test/CodeGen/Hexagon/loop-idiom/nullptr-crash.ll @@ -1,4 +1,5 @@ ; RUN: opt -hexagon-loop-idiom -mtriple hexagon-unknown-elf < %s +; RUN: opt -p hexagon-loop-idiom -mtriple hexagon-unknown-elf < %s ; REQUIRES: asserts target triple = "hexagon" diff --git a/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy-infinite-loop.ll b/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy-infinite-loop.ll index 1934ced7e7ae..74c02d63d54d 100644 --- a/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy-infinite-loop.ll +++ b/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy-infinite-loop.ll @@ -1,4 +1,5 @@ ; RUN: opt -march=hexagon -hexagon-loop-idiom -S < %s | FileCheck %s +; RUN: opt -march=hexagon -p hexagon-loop-idiom -S < %s | FileCheck %s ; CHECK-LABEL: define void @fred ; Check that this test does not crash. diff --git a/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy-long-loop.ll b/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy-long-loop.ll index b25010f2a90f..94b0c96c3d51 100644 --- a/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy-long-loop.ll +++ b/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy-long-loop.ll @@ -1,4 +1,5 @@ ; RUN: opt -march=hexagon -hexagon-loop-idiom -S < %s | FileCheck %s +; RUN: opt -march=hexagon -p hexagon-loop-idiom -S < %s | FileCheck %s ; ; The number of nested selects caused the simplification loop to take ; more than the maximum number of iterations. This caused the compiler diff --git a/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy-shiftconv-fail.ll b/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy-shiftconv-fail.ll index e4b2b5a298ed..a00b1d5876ba 100644 --- a/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy-shiftconv-fail.ll +++ b/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy-shiftconv-fail.ll @@ -1,4 +1,5 @@ ; RUN: opt -march=hexagon -hexagon-loop-idiom -S < %s | FileCheck %s +; RUN: opt -march=hexagon -p hexagon-loop-idiom -S < %s | FileCheck %s ; REQUIRES: asserts ; ; Check for sane output, this used to crash. diff --git a/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy.ll b/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy.ll index 781618e58901..2461e1cfde8d 100644 --- a/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy.ll +++ b/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy.ll @@ -1,5 +1,7 @@ ; RUN: opt -hexagon-loop-idiom < %s -mtriple=hexagon-unknown-unknown -S \ ; RUN: | FileCheck %s +; RUN: opt -p hexagon-loop-idiom < %s -mtriple=hexagon-unknown-unknown -S \ +; RUN: | FileCheck %s target triple = "hexagon" -- GitLab From 691b97c884a15a7eac641ddf67c9f2f30fb4e747 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Sat, 23 Mar 2024 01:02:08 -0700 Subject: [PATCH 138/306] [ELF] Remove zero-value DT_JMPREL when IPLT is present while PLT isn't The zero-value DT_JMPREL is benign but not needed. This is also code simplification available after https://reviews.llvm.org/D65651 --- lld/ELF/SyntheticSections.cpp | 16 ++------------- .../ELF/aarch64-gnu-ifunc-nonpreemptable.s | 12 +++++------ lld/test/ELF/gnu-ifunc-dyntags.s | 8 +++----- lld/test/ELF/ppc32-ifunc-nonpreemptible-pic.s | 8 ++++---- lld/test/ELF/riscv-ifunc-nonpreemptible.s | 20 +++++++++---------- 5 files changed, 25 insertions(+), 39 deletions(-) diff --git a/lld/ELF/SyntheticSections.cpp b/lld/ELF/SyntheticSections.cpp index 248ff6b4a865..10eda17f0cb3 100644 --- a/lld/ELF/SyntheticSections.cpp +++ b/lld/ELF/SyntheticSections.cpp @@ -1285,13 +1285,7 @@ static uint64_t addRelaSz(const RelocationBaseSection &relaDyn) { // output section. When this occurs we cannot just use the OutputSection // Size. Moreover the [DT_JMPREL, DT_JMPREL + DT_PLTRELSZ) is permitted to // overlap with the [DT_RELA, DT_RELA + DT_RELASZ). -static uint64_t addPltRelSz() { - size_t size = in.relaPlt->getSize(); - if (in.relaIplt->getParent() == in.relaPlt->getParent() && - in.relaIplt->name == in.relaPlt->name) - size += in.relaIplt->getSize(); - return size; -} +static uint64_t addPltRelSz() { return in.relaPlt->getSize(); } // Add remaining entries to complete .dynamic contents. template @@ -1407,13 +1401,7 @@ DynamicSection::computeContents() { addInt(config->useAndroidRelrTags ? DT_ANDROID_RELRENT : DT_RELRENT, sizeof(Elf_Relr)); } - // .rel[a].plt section usually consists of two parts, containing plt and - // iplt relocations. It is possible to have only iplt relocations in the - // output. In that case relaPlt is empty and have zero offset, the same offset - // as relaIplt has. And we still want to emit proper dynamic tags for that - // case, so here we always use relaPlt as marker for the beginning of - // .rel[a].plt section. - if (isMain && (in.relaPlt->isNeeded() || in.relaIplt->isNeeded())) { + if (isMain && in.relaPlt->isNeeded()) { addInSec(DT_JMPREL, *in.relaPlt); entries.emplace_back(DT_PLTRELSZ, addPltRelSz()); switch (config->emachine) { diff --git a/lld/test/ELF/aarch64-gnu-ifunc-nonpreemptable.s b/lld/test/ELF/aarch64-gnu-ifunc-nonpreemptable.s index 4f33dde9d1a9..54a27a8cc0d9 100644 --- a/lld/test/ELF/aarch64-gnu-ifunc-nonpreemptable.s +++ b/lld/test/ELF/aarch64-gnu-ifunc-nonpreemptable.s @@ -65,14 +65,14 @@ main: # PIE-EMPTY: # PIE-NEXT: : # PIE-NEXT: 10270: adrp x16, 0x30000 -# PIE-NEXT: 10274: ldr x17, [x16, #896] -# PIE-NEXT: 10278: add x16, x16, #896 -# PIE-NEXT: 1027c: br x17 +# PIE-NEXT: ldr x17, [x16, #832] +# PIE-NEXT: add x16, x16, #832 +# PIE-NEXT: br x17 # PIE-RELOC: .rela.dyn { -# PIE-RELOC-NEXT: 0x30380 R_AARCH64_IRELATIVE - 0x10260 +# PIE-RELOC-NEXT: 0x30340 R_AARCH64_IRELATIVE - 0x10260 # PIE-RELOC-NEXT: } # PIE-RELOC: Hex dump of section '.got.plt': -# NO-APPLY: 0x00030380 00000000 00000000 -# APPLY: 0x00030380 60020100 00000000 +# NO-APPLY: 0x00030340 00000000 00000000 +# APPLY: 0x00030340 60020100 00000000 # PIE-RELOC-EMPTY: diff --git a/lld/test/ELF/gnu-ifunc-dyntags.s b/lld/test/ELF/gnu-ifunc-dyntags.s index fd80dc24f2f8..57a17245a3e8 100644 --- a/lld/test/ELF/gnu-ifunc-dyntags.s +++ b/lld/test/ELF/gnu-ifunc-dyntags.s @@ -9,15 +9,13 @@ # CHECK: Name Size VMA # CHECK: .rela.dyn 00000030 0000000000000248 -# CHECK: .got.plt 00000010 00000000000033b0 +# CHECK: .got.plt 00000010 0000000000003370 # TAGS: Tag Type Name/Value # TAGS: 0x0000000000000007 RELA 0x248 # TAGS: 0x0000000000000008 RELASZ 48 (bytes) -# TAGS: 0x0000000000000017 JMPREL 0x0 -# TAGS: 0x0000000000000002 PLTRELSZ 0 (bytes) -# TAGS: 0x0000000000000003 PLTGOT 0x33B0 -# TAGS: 0x0000000000000014 PLTREL RELA +# TAGS-NOT: JMPREL +# TAGS-NOT: PLTREL # TAGS: Relocations [ # TAGS-NEXT: Section {{.*}} .rela.dyn { diff --git a/lld/test/ELF/ppc32-ifunc-nonpreemptible-pic.s b/lld/test/ELF/ppc32-ifunc-nonpreemptible-pic.s index a93f3cecb0c6..c9a0381b610a 100644 --- a/lld/test/ELF/ppc32-ifunc-nonpreemptible-pic.s +++ b/lld/test/ELF/ppc32-ifunc-nonpreemptible-pic.s @@ -10,16 +10,16 @@ # RUN: llvm-readelf -x .got2 %t | FileCheck --check-prefix=HEX2 %s # RELOC: .rela.dyn { -# RELOC-NEXT: 0x3024C R_PPC_RELATIVE - 0x101A0 -# RELOC-NEXT: 0x30250 R_PPC_IRELATIVE - 0x10188 +# RELOC-NEXT: 0x3022C R_PPC_RELATIVE - 0x101A0 +# RELOC-NEXT: 0x30230 R_PPC_IRELATIVE - 0x10188 # RELOC-NEXT: } # SYM: 000101a0 0 FUNC GLOBAL DEFAULT {{.*}} func # HEX: Hex dump of section '.got2': -# HEX-NEXT: 0x0003024c 00000000 .... +# HEX-NEXT: 0x0003022c 00000000 .... # HEX2: Hex dump of section '.got2': -# HEX2-NEXT: 0x0003024c 000101a0 .... +# HEX2-NEXT: 0x0003022c 000101a0 .... .section .got2,"aw" .long func diff --git a/lld/test/ELF/riscv-ifunc-nonpreemptible.s b/lld/test/ELF/riscv-ifunc-nonpreemptible.s index 21c607545110..eda5548eef8b 100644 --- a/lld/test/ELF/riscv-ifunc-nonpreemptible.s +++ b/lld/test/ELF/riscv-ifunc-nonpreemptible.s @@ -16,11 +16,11 @@ # RUN: llvm-objdump -d --no-show-raw-insn %t.64 | FileCheck --check-prefix=DIS64 %s # RELOC32: .rela.dyn { -# RELOC32-NEXT: 0x3220 R_RISCV_IRELATIVE - 0x117C +# RELOC32-NEXT: 0x3200 R_RISCV_IRELATIVE - 0x117C # RELOC32-NEXT: } # RELOC32-LABEL: Hex dump of section '.got.plt': -# NO-APPLY-RELOC32: 0x00003220 00000000 -# APPLY-RELOC32: 0x00003220 7c110000 +# NO-APPLY-RELOC32: 0x00003200 00000000 +# APPLY-RELOC32: 0x00003200 7c110000 # RELOC32-EMPTY: # SYM32: 0001190 0 FUNC GLOBAL DEFAULT {{.*}} func @@ -30,18 +30,18 @@ # DIS32-NEXT: addi a0, a0, 0x10 # DIS32: Disassembly of section .iplt: # DIS32: : -## 32-bit: &.got.plt[func]-. = 0x3220-0x1190 = 4096*2+144 +## 32-bit: &.got.plt[func]-. = 0x3200-0x1190 = 4096*2+0x70 # DIS32-NEXT: 1190: auipc t3, 0x2 -# DIS32-NEXT: lw t3, 0x90(t3) +# DIS32-NEXT: lw t3, 0x70(t3) # DIS32-NEXT: jalr t1, t3 # DIS32-NEXT: nop # RELOC64: .rela.dyn { -# RELOC64-NEXT: 0x3380 R_RISCV_IRELATIVE - 0x1260 +# RELOC64-NEXT: 0x3340 R_RISCV_IRELATIVE - 0x1260 # RELOC64-NEXT: } # RELOC64-LABEL: Hex dump of section '.got.plt': -# NO-APPLY-RELOC64: 0x00003380 00000000 00000000 -# APPLY-RELOC64: 0x00003380 60120000 00000000 +# NO-APPLY-RELOC64: 0x00003340 00000000 00000000 +# APPLY-RELOC64: 0x00003340 60120000 00000000 # RELOC64-EMPTY: # SYM64: 000000000001270 0 FUNC GLOBAL DEFAULT {{.*}} func @@ -51,9 +51,9 @@ # DIS64-NEXT: addi a0, a0, 0xc # DIS64: Disassembly of section .iplt: # DIS64: : -## 64-bit: &.got.plt[func]-. = 0x3380-0x1270 = 4096*2+272 +## 64-bit: &.got.plt[func]-. = 0x3340-0x1270 = 4096*2+0xd0 # DIS64-NEXT: 1270: auipc t3, 0x2 -# DIS64-NEXT: ld t3, 0x110(t3) +# DIS64-NEXT: ld t3, 0xd0(t3) # DIS64-NEXT: jalr t1, t3 # DIS64-NEXT: nop -- GitLab From 579dc7f8441a8044b92bdfa6f0db2f91301c0eed Mon Sep 17 00:00:00 2001 From: Owen Pan Date: Sat, 23 Mar 2024 02:03:45 -0700 Subject: [PATCH 139/306] [clang-forma] Support `PointerAlignment` for pointers to members (#86253) Fixes #85761. --- clang/lib/Format/TokenAnnotator.cpp | 37 +++++++++++------- clang/unittests/Format/FormatTest.cpp | 38 ++++++++++++------- clang/unittests/Format/QualifierFixerTest.cpp | 36 +++++++++--------- 3 files changed, 66 insertions(+), 45 deletions(-) diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp index a5cafcbacaa5..7757c8ff7639 100644 --- a/clang/lib/Format/TokenAnnotator.cpp +++ b/clang/lib/Format/TokenAnnotator.cpp @@ -4357,9 +4357,11 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line, if (Left.is(tok::kw_auto) && Right.isOneOf(tok::l_paren, tok::l_brace)) return false; + const auto *BeforeLeft = Left.Previous; + // operator co_await(x) - if (Right.is(tok::l_paren) && Left.is(tok::kw_co_await) && Left.Previous && - Left.Previous->is(tok::kw_operator)) { + if (Right.is(tok::l_paren) && Left.is(tok::kw_co_await) && BeforeLeft && + BeforeLeft->is(tok::kw_operator)) { return false; } // co_await (x), co_yield (x), co_return (x) @@ -4394,8 +4396,10 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line, } if (Left.is(tok::colon)) return Left.isNot(TT_ObjCMethodExpr); - if (Left.is(tok::coloncolon)) - return false; + if (Left.is(tok::coloncolon)) { + return Right.is(tok::star) && Right.is(TT_PointerOrReference) && + Style.PointerAlignment != FormatStyle::PAS_Left; + } if (Left.is(tok::less) || Right.isOneOf(tok::greater, tok::less)) { if (Style.Language == FormatStyle::LK_TextProto || (Style.Language == FormatStyle::LK_Proto && @@ -4410,8 +4414,8 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line, return false; } if (Right.is(tok::ellipsis)) { - return Left.Tok.isLiteral() || (Left.is(tok::identifier) && Left.Previous && - Left.Previous->is(tok::kw_case)); + return Left.Tok.isLiteral() || (Left.is(tok::identifier) && BeforeLeft && + BeforeLeft->is(tok::kw_case)); } if (Left.is(tok::l_square) && Right.is(tok::amp)) return Style.SpacesInSquareBrackets; @@ -4479,8 +4483,8 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line, if (Right.is(tok::l_brace) && Right.is(BK_Block)) return true; // for (auto a = 0, b = 0; const auto& c : {1, 2, 3}) - if (Left.Previous && Left.Previous->isTypeOrIdentifier(IsCpp) && - Right.Next && Right.Next->is(TT_RangeBasedForLoopColon)) { + if (BeforeLeft && BeforeLeft->isTypeOrIdentifier(IsCpp) && Right.Next && + Right.Next->is(TT_RangeBasedForLoopColon)) { return getTokenPointerOrReferenceAlignment(Left) != FormatStyle::PAS_Right; } @@ -4502,12 +4506,17 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line, startsWithInitStatement(Line)))) { return false; } - return Left.Previous && !Left.Previous->isOneOf( - tok::l_paren, tok::coloncolon, tok::l_square); + if (!BeforeLeft) + return false; + if (BeforeLeft->is(tok::coloncolon)) { + return Left.is(tok::star) && + Style.PointerAlignment != FormatStyle::PAS_Right; + } + return !BeforeLeft->isOneOf(tok::l_paren, tok::l_square); } // Ensure right pointer alignment with ellipsis e.g. int *...P - if (Left.is(tok::ellipsis) && Left.Previous && - Left.Previous->isPointerOrReference()) { + if (Left.is(tok::ellipsis) && BeforeLeft && + BeforeLeft->isPointerOrReference()) { return Style.PointerAlignment != FormatStyle::PAS_Right; } @@ -4669,13 +4678,13 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line, return Style.SpaceBeforeParensOptions.AfterFunctionDefinitionName || spaceRequiredBeforeParens(Right); } - if (!Left.Previous || !Left.Previous->isOneOf(tok::period, tok::arrow)) { + if (!BeforeLeft || !BeforeLeft->isOneOf(tok::period, tok::arrow)) { if (Left.isOneOf(tok::kw_try, Keywords.kw___except, tok::kw_catch)) { return Style.SpaceBeforeParensOptions.AfterControlStatements || spaceRequiredBeforeParens(Right); } if (Left.isOneOf(tok::kw_new, tok::kw_delete)) { - return ((!Line.MightBeFunctionDecl || !Left.Previous) && + return ((!Line.MightBeFunctionDecl || !BeforeLeft) && Style.SpaceBeforeParens != FormatStyle::SBPO_Never) || spaceRequiredBeforeParens(Right); } diff --git a/clang/unittests/Format/FormatTest.cpp b/clang/unittests/Format/FormatTest.cpp index bea989c8c306..cf8d6ab691d9 100644 --- a/clang/unittests/Format/FormatTest.cpp +++ b/clang/unittests/Format/FormatTest.cpp @@ -3621,8 +3621,8 @@ TEST_F(FormatTest, FormatsClasses) { " : public aaaaaaaaaaaaaaaaaaa {};"); verifyFormat("template \n" - "struct Aaaaaaaaaaaaaaaaa\n" - " : Aaaaaaaaaaaaaaaaa {};"); + "struct Aaaaaaaaaaaaaaaaa\n" + " : Aaaaaaaaaaaaaaaaa {};"); verifyFormat("class ::A::B {};"); } @@ -11034,10 +11034,10 @@ TEST_F(FormatTest, UnderstandsBinaryOperators) { } TEST_F(FormatTest, UnderstandsPointersToMembers) { - verifyFormat("int A::*x;"); - verifyFormat("int (S::*func)(void *);"); - verifyFormat("void f() { int (S::*func)(void *); }"); - verifyFormat("typedef bool *(Class::*Member)() const;"); + verifyFormat("int A:: *x;"); + verifyFormat("int (S:: *func)(void *);"); + verifyFormat("void f() { int (S:: *func)(void *); }"); + verifyFormat("typedef bool *(Class:: *Member)() const;"); verifyFormat("void f() {\n" " (a->*f)();\n" " a->*x;\n" @@ -11052,9 +11052,19 @@ TEST_F(FormatTest, UnderstandsPointersToMembers) { verifyFormat( "(aaaaaaaaaa->*bbbbbbb)(\n" " aaaaaaaaaaaaaaaaaaaaaaaaaaa(aaaaaaaaaaaaaaaaaaaaaaaaaaa));"); + FormatStyle Style = getLLVMStyle(); + EXPECT_EQ(Style.PointerAlignment, FormatStyle::PAS_Right); + verifyFormat("typedef bool *(Class:: *Member)() const;", Style); + verifyFormat("void f(int A:: *p) { int A:: *v = &A::B; }", Style); + Style.PointerAlignment = FormatStyle::PAS_Left; - verifyFormat("typedef bool* (Class::*Member)() const;", Style); + verifyFormat("typedef bool* (Class::* Member)() const;", Style); + verifyFormat("void f(int A::* p) { int A::* v = &A::B; }", Style); + + Style.PointerAlignment = FormatStyle::PAS_Middle; + verifyFormat("typedef bool * (Class:: * Member)() const;", Style); + verifyFormat("void f(int A:: * p) { int A:: * v = &A::B; }", Style); } TEST_F(FormatTest, UnderstandsUnaryOperators) { @@ -12386,7 +12396,7 @@ TEST_F(FormatTest, FormatsFunctionTypes) { verifyFormat("int (*func)(void *);"); verifyFormat("void f() { int (*func)(void *); }"); verifyFormat("template \n" - "using MyCallback = void (CallbackClass::*)(SomeObject *Data);"); + "using Callback = void (CallbackClass:: *)(SomeObject *Data);"); verifyGoogleFormat("A;"); verifyGoogleFormat("void* (*a)(int);"); @@ -19149,13 +19159,13 @@ TEST_F(FormatTest, AlignConsecutiveDeclarations) { "int bbbbbbb = 0;", Alignment); // http://llvm.org/PR68079 - verifyFormat("using Fn = int (A::*)();\n" - "using RFn = int (A::*)() &;\n" - "using RRFn = int (A::*)() &&;", + verifyFormat("using Fn = int (A:: *)();\n" + "using RFn = int (A:: *)() &;\n" + "using RRFn = int (A:: *)() &&;", Alignment); - verifyFormat("using Fn = int (A::*)();\n" - "using RFn = int *(A::*)() &;\n" - "using RRFn = double (A::*)() &&;", + verifyFormat("using Fn = int (A:: *)();\n" + "using RFn = int *(A:: *)() &;\n" + "using RRFn = double (A:: *)() &&;", Alignment); // PAS_Right diff --git a/clang/unittests/Format/QualifierFixerTest.cpp b/clang/unittests/Format/QualifierFixerTest.cpp index 43476aea6633..792d8f3c3a98 100644 --- a/clang/unittests/Format/QualifierFixerTest.cpp +++ b/clang/unittests/Format/QualifierFixerTest.cpp @@ -305,7 +305,7 @@ TEST_F(QualifierFixerTest, RightQualifier) { verifyFormat("Foo inline static const;", "Foo inline const static;", Style); verifyFormat("Foo inline static const;", Style); - verifyFormat("Foo::Bar const volatile A::*;", + verifyFormat("Foo::Bar const volatile A:: *;", "volatile const Foo::Bar A::*;", Style); @@ -523,14 +523,15 @@ TEST_F(QualifierFixerTest, RightQualifier) { verifyFormat("const INTPTR a;", Style); // Pointers to members - verifyFormat("int S::*a;", Style); - verifyFormat("int const S::*a;", "const int S:: *a;", Style); - verifyFormat("int const S::*const a;", "const int S::* const a;", Style); - verifyFormat("int A::*const A::*p1;", Style); - verifyFormat("float (C::*p)(int);", Style); - verifyFormat("float (C::*const p)(int);", Style); - verifyFormat("float (C::*p)(int) const;", Style); - verifyFormat("float const (C::*p)(int);", "const float (C::*p)(int);", Style); + verifyFormat("int S:: *a;", Style); + verifyFormat("int const S:: *a;", "const int S:: *a;", Style); + verifyFormat("int const S:: *const a;", "const int S::* const a;", Style); + verifyFormat("int A:: *const A:: *p1;", Style); + verifyFormat("float (C:: *p)(int);", Style); + verifyFormat("float (C:: *const p)(int);", Style); + verifyFormat("float (C:: *p)(int) const;", Style); + verifyFormat("float const (C:: *p)(int);", "const float (C::*p)(int);", + Style); } TEST_F(QualifierFixerTest, LeftQualifier) { @@ -830,14 +831,15 @@ TEST_F(QualifierFixerTest, LeftQualifier) { verifyFormat("INTPTR const a;", Style); // Pointers to members - verifyFormat("int S::*a;", Style); - verifyFormat("const int S::*a;", "int const S:: *a;", Style); - verifyFormat("const int S::*const a;", "int const S::* const a;", Style); - verifyFormat("int A::*const A::*p1;", Style); - verifyFormat("float (C::*p)(int);", Style); - verifyFormat("float (C::*const p)(int);", Style); - verifyFormat("float (C::*p)(int) const;", Style); - verifyFormat("const float (C::*p)(int);", "float const (C::*p)(int);", Style); + verifyFormat("int S:: *a;", Style); + verifyFormat("const int S:: *a;", "int const S:: *a;", Style); + verifyFormat("const int S:: *const a;", "int const S::* const a;", Style); + verifyFormat("int A:: *const A:: *p1;", Style); + verifyFormat("float (C:: *p)(int);", Style); + verifyFormat("float (C:: *const p)(int);", Style); + verifyFormat("float (C:: *p)(int) const;", Style); + verifyFormat("const float (C:: *p)(int);", "float const (C::*p)(int);", + Style); } TEST_F(QualifierFixerTest, ConstVolatileQualifiersOrder) { -- GitLab From 47423e9827abfdcc6b10ce41618965861b0e69a4 Mon Sep 17 00:00:00 2001 From: Owen Pan Date: Sat, 23 Mar 2024 02:24:56 -0700 Subject: [PATCH 140/306] [clang-format][NFC] Clean up IsQualifiedPointerOrReference in TokenAnnotator --- clang/lib/Format/TokenAnnotator.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp index 7757c8ff7639..4c83a7a3a323 100644 --- a/clang/lib/Format/TokenAnnotator.cpp +++ b/clang/lib/Format/TokenAnnotator.cpp @@ -2753,10 +2753,9 @@ private: } // Heuristically try to determine whether the parentheses contain a type. - auto IsQualifiedPointerOrReference = [this](FormatToken *T) { + auto IsQualifiedPointerOrReference = [](FormatToken *T, bool IsCpp) { // This is used to handle cases such as x = (foo *const)&y; assert(!T->isTypeName(IsCpp) && "Should have already been checked"); - (void)IsCpp; // Avoid -Wunused-lambda-capture when assertion is disabled. // Strip trailing qualifiers such as const or volatile when checking // whether the parens could be a cast to a pointer/reference type. while (T) { @@ -2789,7 +2788,7 @@ private: !Tok.Previous || Tok.Previous->isOneOf(TT_TemplateCloser, TT_TypeDeclarationParen) || Tok.Previous->isTypeName(IsCpp) || - IsQualifiedPointerOrReference(Tok.Previous); + IsQualifiedPointerOrReference(Tok.Previous, IsCpp); bool ParensCouldEndDecl = Tok.Next->isOneOf(tok::equal, tok::semi, tok::l_brace, tok::greater); if (ParensAreType && !ParensCouldEndDecl) -- GitLab From f317fd266c184bc8f9f4d7e8e03c043d3406333a Mon Sep 17 00:00:00 2001 From: Muhammad Omair Javaid Date: Sat, 23 Mar 2024 15:30:26 +0500 Subject: [PATCH 141/306] Revert "[mlir][SVE] Add e2e for 1D depthwise WC convolution (#85225)" This reverts commit 01b1b0c1f728e2c2639edc654424f50830295989. Breaks following AArch64 SVE buildbots: https://lab.llvm.org/buildbot/#/builders/184/builds/11363 https://lab.llvm.org/buildbot/#/builders/176/builds/9331 --- .../Linalg/CPU/ArmSVE/1d-depthwise-conv.mlir | 60 ------------------- 1 file changed, 60 deletions(-) delete mode 100644 mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/1d-depthwise-conv.mlir diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/1d-depthwise-conv.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/1d-depthwise-conv.mlir deleted file mode 100644 index 57d69383c2de..000000000000 --- a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/1d-depthwise-conv.mlir +++ /dev/null @@ -1,60 +0,0 @@ -// DEFINE: %{compile} = mlir-opt %s \ -// DEFINE: -transform-interpreter -test-transform-dialect-erase-schedule \ -// DEFINE: -one-shot-bufferize="bufferize-function-boundaries" -lower-vector-mask -cse -canonicalize -convert-vector-to-scf -arm-sve-legalize-vector-storage \ -// DEFINE: -convert-vector-to-llvm="enable-arm-sve" -test-lower-to-llvm -o %t -// DEFINE: %{entry_point} = conv -// DEFINE: %{run} = %mcr_aarch64_cmd %t -e %{entry_point} -entry-point-result=void --march=aarch64 --mattr="+sve"\ -// DEFINE: -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils - -// RUN: %{compile} | %{run} | FileCheck %s - -func.func @conv() { - // Define input/output tensors - %input_init = tensor.empty() : tensor<1x8x6xi32> - %output_init = tensor.empty() : tensor<1x7x6xi32> - - %five = arith.constant 5 : i32 - %zero = arith.constant 0 : i32 - %input = linalg.fill ins(%five : i32) outs(%input_init : tensor<1x8x6xi32>) -> tensor<1x8x6xi32> - %output = linalg.fill ins(%zero : i32) outs(%output_init : tensor<1x7x6xi32>) -> tensor<1x7x6xi32> - - // Define the filter tensor - %filter = arith.constant dense<[ - [ 1, 2, 3, 4, 5, 6], - [ 11, 12, 13, 14, 15, 16] - ]> : tensor<2x6xi32> - - // static sizes -> dynamic sizes - %input_dyn = tensor.cast %input_init : tensor<1x8x6xi32> to tensor<1x8x?xi32> - %output_dyn = tensor.cast %output : tensor<1x7x6xi32> to tensor<1x7x?xi32> - %filter_dyn = tensor.cast %filter : tensor<2x6xi32> to tensor<2x?xi32> - - // Run the convolution - %res = linalg.depthwise_conv_1d_nwc_wc - ins(%input_dyn, %filter_dyn : tensor<1x8x?xi32>, tensor<2x?xi32>) - outs(%output_dyn : tensor<1x7x?xi32>) -> tensor<1x7x?xi32> - - // Print the results - // CHECK: SVE: START OF TEST OUTPUT - vector.print str "SVE: START OF TEST OUTPUT\n" - - // CHECK-NEXT: Unranked Memref base@ = {{.*}} rank = 3 offset = 0 sizes = [1, 7, 6] strides = [42, 6, 1] data = - // CHECK-COUNT-7: [60, 70, 80, 90, 100, 110] - %xf = tensor.cast %res : tensor<1x7x?xi32> to tensor<*xi32> - call @printMemrefI32(%xf) : (tensor<*xi32>) -> () - - // CHECK-NEXT: SVE: END OF TEST OUTPUT - vector.print str "SVE: END OF TEST OUTPUT\n" - - return -} - -module attributes {transform.with_named_sequence} { - transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) { - %0 = transform.structured.match ops{["linalg.depthwise_conv_1d_nwc_wc"]} in %arg0 : (!transform.any_op) -> !transform.any_op - transform.structured.vectorize %0 vector_sizes [1, 7, [8], 2] : !transform.any_op - transform.yield - } -} - -func.func private @printMemrefI32(%ptr : tensor<*xi32>) attributes { llvm.emit_c_interface } -- GitLab From d7c672834ec863b458af8ca493157e1e31aaf480 Mon Sep 17 00:00:00 2001 From: XChy Date: Sat, 23 Mar 2024 19:00:42 +0800 Subject: [PATCH 142/306] [CodeGen][NFC] Update tests in AArch64/and-sink.ll --- llvm/test/CodeGen/AArch64/and-sink.ll | 60 +++++++++++++++++++++------ 1 file changed, 47 insertions(+), 13 deletions(-) diff --git a/llvm/test/CodeGen/AArch64/and-sink.ll b/llvm/test/CodeGen/AArch64/and-sink.ll index 4d085869de24..f298a55dab72 100644 --- a/llvm/test/CodeGen/AArch64/and-sink.ll +++ b/llvm/test/CodeGen/AArch64/and-sink.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 ; RUN: llc -mtriple=aarch64-linux-gnu -verify-machineinstrs < %s | FileCheck %s ; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=aarch64-linux %s | FileCheck --check-prefix=CHECK-CGP %s ; RUN: opt -S -passes='require,function(codegenprepare)' -cgpp-huge-func=0 -mtriple=aarch64-linux %s | FileCheck --check-prefix=CHECK-CGP %s @@ -9,9 +10,18 @@ ; Test that and is sunk into cmp block to form tbz. define dso_local i32 @and_sink1(i32 %a, i1 %c) { ; CHECK-LABEL: and_sink1: -; CHECK: tbz w1, #0 -; CHECK: str wzr, [x{{[0-9]+}}, :lo12:A] -; CHECK: tbnz {{w[0-9]+}}, #2 +; CHECK: // %bb.0: +; CHECK-NEXT: tbz w1, #0, .LBB0_3 +; CHECK-NEXT: // %bb.1: // %bb0 +; CHECK-NEXT: adrp x8, A +; CHECK-NEXT: str wzr, [x8, :lo12:A] +; CHECK-NEXT: tbnz w0, #2, .LBB0_3 +; CHECK-NEXT: // %bb.2: +; CHECK-NEXT: mov w0, #1 // =0x1 +; CHECK-NEXT: ret +; CHECK-NEXT: .LBB0_3: // %bb2 +; CHECK-NEXT: mov w0, wzr +; CHECK-NEXT: ret ; CHECK-CGP-LABEL: @and_sink1( ; CHECK-CGP-NOT: and i32 @@ -35,12 +45,30 @@ bb2: ; Test that both 'and' and cmp get sunk to form tbz. define dso_local i32 @and_sink2(i32 %a, i1 %c, i1 %c2) { ; CHECK-LABEL: and_sink2: -; CHECK: str wzr, [x{{[0-9]+}}, :lo12:A] -; CHECK: tbz w1, #0 -; CHECK: str wzr, [x{{[0-9]+}}, :lo12:B] -; CHECK: tbz w2, #0 -; CHECK: str wzr, [x{{[0-9]+}}, :lo12:C] -; CHECK: tbnz {{w[0-9]+}}, #2 +; CHECK: // %bb.0: +; CHECK-NEXT: mov w8, wzr +; CHECK-NEXT: adrp x9, A +; CHECK-NEXT: str wzr, [x9, :lo12:A] +; CHECK-NEXT: tbz w1, #0, .LBB1_5 +; CHECK-NEXT: // %bb.1: // %bb0.preheader +; CHECK-NEXT: adrp x8, B +; CHECK-NEXT: adrp x9, C +; CHECK-NEXT: .LBB1_2: // %bb0 +; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: str wzr, [x8, :lo12:B] +; CHECK-NEXT: tbz w2, #0, .LBB1_6 +; CHECK-NEXT: // %bb.3: // %bb1 +; CHECK-NEXT: // in Loop: Header=BB1_2 Depth=1 +; CHECK-NEXT: str wzr, [x9, :lo12:C] +; CHECK-NEXT: tbnz w0, #2, .LBB1_2 +; CHECK-NEXT: // %bb.4: +; CHECK-NEXT: mov w8, #1 // =0x1 +; CHECK-NEXT: .LBB1_5: // %common.ret +; CHECK-NEXT: mov w0, w8 +; CHECK-NEXT: ret +; CHECK-NEXT: .LBB1_6: +; CHECK-NEXT: mov w0, wzr +; CHECK-NEXT: ret ; CHECK-CGP-LABEL: @and_sink2( ; CHECK-CGP-NOT: and i32 @@ -71,10 +99,16 @@ bb3: ; Test that 'and' is not sunk since cbz is a better alternative. define dso_local i32 @and_sink3(i32 %a) { ; CHECK-LABEL: and_sink3: -; CHECK: and [[REG:w[0-9]+]], w0, #0x3 -; CHECK: [[LOOP:.L[A-Z0-9_]+]]: -; CHECK: str wzr, [x{{[0-9]+}}, :lo12:A] -; CHECK: cbz [[REG]], [[LOOP]] +; CHECK: // %bb.0: +; CHECK-NEXT: adrp x8, A +; CHECK-NEXT: and w9, w0, #0x3 +; CHECK-NEXT: .LBB2_1: // %bb0 +; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: str wzr, [x8, :lo12:A] +; CHECK-NEXT: cbz w9, .LBB2_1 +; CHECK-NEXT: // %bb.2: // %bb2 +; CHECK-NEXT: mov w0, wzr +; CHECK-NEXT: ret ; CHECK-CGP-LABEL: @and_sink3( ; CHECK-CGP-NEXT: and i32 -- GitLab From d365a45cb3eaa640b09874fb7984a6a69683c773 Mon Sep 17 00:00:00 2001 From: Evgenii Kudriashov Date: Sat, 23 Mar 2024 15:12:44 +0300 Subject: [PATCH 143/306] [GlobalISel] Introduce G_TRAP, G_DEBUGTRAP, G_UBSANTRAP (#84941) Here we introduce three new GMIR instructions to cover a set of trap intrinsics. The idea behind it is that generic intrinsics shouldn't be used with G_INTRINSIC opcode. These new instructions can match perfectly with existing trap ISD nodes. It allows X86, AArch64, RISCV and Mips to reuse SelectionDAG patterns for selection and avoid manual selection. However AMDGPU is an exception. It selects traps during legalization regardless SelectionDAG or GlobalISel. Since there are not many places where traps are used, this change attempts to clean up all the usages of G_INTRINSIC with trap intrinsics. So, there is no stage when both G_TRAP and G_INTRINSIC_W_SIDE_EFFECTS(@llvm.trap) are allowed. --- llvm/docs/GlobalISel/GenericOpcode.rst | 19 +++++ llvm/docs/LangRef.rst | 6 ++ .../llvm/CodeGen/GlobalISel/IRTranslator.h | 4 + .../CodeGen/GlobalISel/MachineIRBuilder.h | 5 ++ llvm/include/llvm/Support/TargetOpcodes.def | 5 ++ llvm/include/llvm/Target/GenericOpcodes.td | 22 ++++++ .../Target/GlobalISel/SelectionDAGCompat.td | 3 + llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 47 ++++++++---- llvm/lib/CodeGen/MachineVerifier.cpp | 11 +++ llvm/lib/Target/AArch64/AArch64InstrInfo.td | 3 + .../GISel/AArch64InstructionSelector.cpp | 20 ++--- .../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 23 +++--- llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h | 8 +- llvm/lib/Target/Mips/MipsLegalizerInfo.cpp | 8 -- .../RISCV/GISel/RISCVInstructionSelector.cpp | 27 ------- .../X86/GISel/X86InstructionSelector.cpp | 19 ----- .../GlobalISel/irtranslator-unreachable.ll | 2 +- .../AArch64/GlobalISel/legalize-exceptions.ll | 2 +- .../GlobalISel/legalizer-info-validation.mir | 9 +++ .../AArch64/GlobalISel/select-trap.mir | 2 +- .../AArch64/GlobalISel/uaddo-8-16-bits.mir | 52 ++++++------- .../AMDGPU/GlobalISel/legalize-trap.mir | 4 +- .../trap.mir | 9 ++- .../GlobalISel/instruction-select/trap.mir | 4 +- .../X86/GlobalISel/x86-select-trap.mir | 2 +- llvm/test/CodeGen/X86/isel-traps.ll | 73 +++++++++++++++++++ .../test/MachineVerifier/test_g_ubsantrap.mir | 18 +++++ 27 files changed, 276 insertions(+), 131 deletions(-) rename llvm/test/CodeGen/Mips/GlobalISel/{legalizer => instruction-select}/trap.mir (55%) create mode 100644 llvm/test/CodeGen/X86/isel-traps.ll create mode 100644 llvm/test/MachineVerifier/test_g_ubsantrap.mir diff --git a/llvm/docs/GlobalISel/GenericOpcode.rst b/llvm/docs/GlobalISel/GenericOpcode.rst index ac6217d08e6a..cae2c21b80d7 100644 --- a/llvm/docs/GlobalISel/GenericOpcode.rst +++ b/llvm/docs/GlobalISel/GenericOpcode.rst @@ -939,6 +939,25 @@ The _CONVERGENT variant corresponds to an LLVM IR intrinsic marked `convergent`. Unlike SelectionDAG, there is no _VOID variant. Both of these are permitted to have zero, one, or multiple results. +G_TRAP, G_DEBUGTRAP, G_UBSANTRAP +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Represents :ref:`llvm.trap `, :ref:`llvm.debugtrap ` +and :ref:`llvm.ubsantrap ` that generate a target dependent +trap instructions. + +.. code-block:: none + + G_TRAP + +.. code-block:: none + + G_DEBUGTRAP + +.. code-block:: none + + G_UBSANTRAP 12 + Variadic Arguments ------------------ diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst index 8bc1cab01bf0..391fd2d960dd 100644 --- a/llvm/docs/LangRef.rst +++ b/llvm/docs/LangRef.rst @@ -26926,6 +26926,8 @@ Arguments: The argument should be an MDTuple containing any number of MDStrings. +.. _llvm.trap: + '``llvm.trap``' Intrinsic ^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -26953,6 +26955,8 @@ This intrinsic is lowered to the target dependent trap instruction. If the target does not have a trap instruction, this intrinsic will be lowered to a call of the ``abort()`` function. +.. _llvm.debugtrap: + '``llvm.debugtrap``' Intrinsic ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -26980,6 +26984,8 @@ This intrinsic is lowered to code which is intended to cause an execution trap with the intention of requesting the attention of a debugger. +.. _llvm.ubsantrap: + '``llvm.ubsantrap``' Intrinsic ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h index 6ae7c1440907..5f28908e998a 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h @@ -243,6 +243,10 @@ private: bool translateMemFunc(const CallInst &CI, MachineIRBuilder &MIRBuilder, unsigned Opcode); + /// Translate an LLVM trap intrinsic (trap, debugtrap, ubsantrap). + bool translateTrap(const CallInst &U, MachineIRBuilder &MIRBuilder, + unsigned Opcode); + // Translate @llvm.experimental.vector.interleave2 and // @llvm.experimental.vector.deinterleave2 intrinsics for fixed-width vector // types into vector shuffles. diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h index aaa81342845b..3ba036ae713f 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h @@ -2113,6 +2113,11 @@ public: DstMMO, SrcMMO); } + /// Build and insert G_TRAP or G_DEBUGTRAP + MachineInstrBuilder buildTrap(bool Debug = false) { + return buildInstr(Debug ? TargetOpcode::G_DEBUGTRAP : TargetOpcode::G_TRAP); + } + /// Build and insert \p Dst = G_SBFX \p Src, \p LSB, \p Width. MachineInstrBuilder buildSbfx(const DstOp &Dst, const SrcOp &Src, const SrcOp &LSB, const SrcOp &Width) { diff --git a/llvm/include/llvm/Support/TargetOpcodes.def b/llvm/include/llvm/Support/TargetOpcodes.def index 899eaad5842a..5765926d6d93 100644 --- a/llvm/include/llvm/Support/TargetOpcodes.def +++ b/llvm/include/llvm/Support/TargetOpcodes.def @@ -837,6 +837,11 @@ HANDLE_TARGET_OPCODE(G_MEMMOVE) HANDLE_TARGET_OPCODE(G_MEMSET) HANDLE_TARGET_OPCODE(G_BZERO) +/// llvm.trap, llvm.debugtrap and llvm.ubsantrap intrinsics +HANDLE_TARGET_OPCODE(G_TRAP) +HANDLE_TARGET_OPCODE(G_DEBUGTRAP) +HANDLE_TARGET_OPCODE(G_UBSANTRAP) + /// Vector reductions HANDLE_TARGET_OPCODE(G_VECREDUCE_SEQ_FADD) HANDLE_TARGET_OPCODE(G_VECREDUCE_SEQ_FMUL) diff --git a/llvm/include/llvm/Target/GenericOpcodes.td b/llvm/include/llvm/Target/GenericOpcodes.td index 67d405ba96fa..d0f471eb29b6 100644 --- a/llvm/include/llvm/Target/GenericOpcodes.td +++ b/llvm/include/llvm/Target/GenericOpcodes.td @@ -1575,6 +1575,28 @@ def G_BZERO : GenericInstruction { let mayStore = true; } +//------------------------------------------------------------------------------ +// Trap intrinsics +//------------------------------------------------------------------------------ +def G_TRAP : GenericInstruction { + let OutOperandList = (outs); + let InOperandList = (ins); + let hasSideEffects = true; + let mayStore = true; +} + +def G_DEBUGTRAP : GenericInstruction { + let OutOperandList = (outs); + let InOperandList = (ins); + let hasSideEffects = true; +} + +def G_UBSANTRAP : GenericInstruction { + let OutOperandList = (outs); + let InOperandList = (ins i8imm:$kind); + let hasSideEffects = true; +} + //------------------------------------------------------------------------------ // Bitfield extraction. //------------------------------------------------------------------------------ diff --git a/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td b/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td index b1f3c500a1b6..4bb9929e2cf8 100644 --- a/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td +++ b/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td @@ -250,6 +250,9 @@ def : GINodeEquiv; def : GINodeEquiv; def : GINodeEquiv; def : GINodeEquiv; +def : GINodeEquiv; +def : GINodeEquiv; +def : GINodeEquiv; // Specifies the GlobalISel equivalents for SelectionDAG's ComplexPattern. // Should be used on defs that subclass GIComplexOperandMatcher<>. diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp index 757af3b1c4fe..ef95ceec18a1 100644 --- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp +++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -1771,6 +1771,32 @@ bool IRTranslator::translateMemFunc(const CallInst &CI, return true; } +bool IRTranslator::translateTrap(const CallInst &CI, + MachineIRBuilder &MIRBuilder, + unsigned Opcode) { + StringRef TrapFuncName = + CI.getAttributes().getFnAttr("trap-func-name").getValueAsString(); + if (TrapFuncName.empty()) { + if (Opcode == TargetOpcode::G_UBSANTRAP) { + uint64_t Code = cast(CI.getOperand(0))->getZExtValue(); + MIRBuilder.buildInstr(Opcode, {}, ArrayRef{Code}); + } else { + MIRBuilder.buildInstr(Opcode); + } + return true; + } + + CallLowering::CallLoweringInfo Info; + if (Opcode == TargetOpcode::G_UBSANTRAP) + Info.OrigArgs.push_back({getOrCreateVRegs(*CI.getArgOperand(0)), + CI.getArgOperand(0)->getType(), 0}); + + Info.Callee = MachineOperand::CreateES(TrapFuncName.data()); + Info.CB = &CI; + Info.OrigRet = {Register(), Type::getVoidTy(CI.getContext()), 0}; + return CLI->lowerCall(MIRBuilder, Info); +} + bool IRTranslator::translateVectorInterleave2Intrinsic( const CallInst &CI, MachineIRBuilder &MIRBuilder) { assert(CI.getIntrinsicID() == Intrinsic::experimental_vector_interleave2 && @@ -2459,22 +2485,11 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID, return true; } case Intrinsic::trap: + return translateTrap(CI, MIRBuilder, TargetOpcode::G_TRAP); case Intrinsic::debugtrap: - case Intrinsic::ubsantrap: { - StringRef TrapFuncName = - CI.getAttributes().getFnAttr("trap-func-name").getValueAsString(); - if (TrapFuncName.empty()) - break; // Use the default handling. - CallLowering::CallLoweringInfo Info; - if (ID == Intrinsic::ubsantrap) { - Info.OrigArgs.push_back({getOrCreateVRegs(*CI.getArgOperand(0)), - CI.getArgOperand(0)->getType(), 0}); - } - Info.Callee = MachineOperand::CreateES(TrapFuncName.data()); - Info.CB = &CI; - Info.OrigRet = {Register(), Type::getVoidTy(CI.getContext()), 0}; - return CLI->lowerCall(MIRBuilder, Info); - } + return translateTrap(CI, MIRBuilder, TargetOpcode::G_DEBUGTRAP); + case Intrinsic::ubsantrap: + return translateTrap(CI, MIRBuilder, TargetOpcode::G_UBSANTRAP); case Intrinsic::amdgcn_cs_chain: return translateCallBase(CI, MIRBuilder); case Intrinsic::fptrunc_round: { @@ -3047,7 +3062,7 @@ bool IRTranslator::translateUnreachable(const User &U, MachineIRBuilder &MIRBuil } } - MIRBuilder.buildIntrinsic(Intrinsic::trap, ArrayRef()); + MIRBuilder.buildTrap(); return true; } diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp index 005efe48ac0c..9a1498d4070e 100644 --- a/llvm/lib/CodeGen/MachineVerifier.cpp +++ b/llvm/lib/CodeGen/MachineVerifier.cpp @@ -1867,6 +1867,17 @@ void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) { break; } + case TargetOpcode::G_UBSANTRAP: { + const MachineOperand &KindOp = MI->getOperand(0); + if (!MI->getOperand(0).isImm()) { + report("Crash kind must be an immediate", &KindOp, 0); + break; + } + int64_t Kind = MI->getOperand(0).getImm(); + if (!isInt<8>(Kind)) + report("Crash kind must be 8 bit wide", &KindOp, 0); + break; + } case TargetOpcode::G_VECREDUCE_SEQ_FADD: case TargetOpcode::G_VECREDUCE_SEQ_FMUL: { LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td index b4b975cce007..b1f514f75207 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td @@ -8436,6 +8436,9 @@ def ubsan_trap_xform : SDNodeXFormgetTargetConstant(N->getZExtValue() | ('U' << 8), SDLoc(N), MVT::i32); }]>; +def gi_ubsan_trap_xform : GICustomOperandRenderer<"renderUbsanTrap">, + GISDNodeXFormEquiv; + def ubsan_trap_imm : TImmLeaf(Imm); }], ubsan_trap_xform>; diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp index 677dd0b502b9..a8f2c45279e6 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp +++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp @@ -479,6 +479,8 @@ private: int OpIdx = -1) const; void renderLogicalImm64(MachineInstrBuilder &MIB, const MachineInstr &I, int OpIdx = -1) const; + void renderUbsanTrap(MachineInstrBuilder &MIB, const MachineInstr &MI, + int OpIdx) const; void renderFPImm16(MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx = -1) const; void renderFPImm32(MachineInstrBuilder &MIB, const MachineInstr &MI, @@ -6159,16 +6161,6 @@ bool AArch64InstructionSelector::selectIntrinsicWithSideEffects( constrainSelectedInstRegOperands(*NewI, TII, TRI, RBI); break; } - case Intrinsic::trap: - MIB.buildInstr(AArch64::BRK, {}, {}).addImm(1); - break; - case Intrinsic::debugtrap: - MIB.buildInstr(AArch64::BRK, {}, {}).addImm(0xF000); - break; - case Intrinsic::ubsantrap: - MIB.buildInstr(AArch64::BRK, {}, {}) - .addImm(I.getOperand(1).getImm() | ('U' << 8)); - break; case Intrinsic::aarch64_neon_ld1x2: { LLT Ty = MRI.getType(I.getOperand(0).getReg()); unsigned Opc = 0; @@ -7663,6 +7655,14 @@ void AArch64InstructionSelector::renderLogicalImm64( MIB.addImm(Enc); } +void AArch64InstructionSelector::renderUbsanTrap(MachineInstrBuilder &MIB, + const MachineInstr &MI, + int OpIdx) const { + assert(MI.getOpcode() == TargetOpcode::G_UBSANTRAP && OpIdx == 0 && + "Expected G_UBSANTRAP"); + MIB.addImm(MI.getOperand(0).getImm() | ('U' << 8)); +} + void AArch64InstructionSelector::renderFPImm16(MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const { diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp index 90872516dd6d..e55d1de01b4f 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp @@ -2030,6 +2030,8 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_, getActionDefinitionsBuilder({G_MEMCPY, G_MEMCPY_INLINE, G_MEMMOVE, G_MEMSET}) .lower(); + getActionDefinitionsBuilder({G_TRAP, G_DEBUGTRAP}).custom(); + getActionDefinitionsBuilder({G_VASTART, G_VAARG, G_BRJT, G_JUMP_TABLE, G_INDEXED_LOAD, G_INDEXED_SEXTLOAD, G_INDEXED_ZEXTLOAD, G_INDEXED_STORE}) @@ -2134,6 +2136,10 @@ bool AMDGPULegalizerInfo::legalizeCustom( return legalizeGetFPEnv(MI, MRI, B); case TargetOpcode::G_SET_FPENV: return legalizeSetFPEnv(MI, MRI, B); + case TargetOpcode::G_TRAP: + return legalizeTrap(MI, MRI, B); + case TargetOpcode::G_DEBUGTRAP: + return legalizeDebugTrap(MI, MRI, B); default: return false; } @@ -2925,7 +2931,7 @@ bool AMDGPULegalizerInfo::legalizeGlobalValue( // functions that use local objects. However, if these dead functions are // not eliminated, we don't want a compile time error. Just emit a warning // and a trap, since there should be no callable path here. - B.buildIntrinsic(Intrinsic::trap, ArrayRef()); + B.buildTrap(); B.buildUndef(DstReg); MI.eraseFromParent(); return true; @@ -6618,9 +6624,9 @@ bool AMDGPULegalizerInfo::legalizeSBufferLoad(LegalizerHelper &Helper, } // TODO: Move to selection -bool AMDGPULegalizerInfo::legalizeTrapIntrinsic(MachineInstr &MI, - MachineRegisterInfo &MRI, - MachineIRBuilder &B) const { +bool AMDGPULegalizerInfo::legalizeTrap(MachineInstr &MI, + MachineRegisterInfo &MRI, + MachineIRBuilder &B) const { if (!ST.isTrapHandlerEnabled() || ST.getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbi::AMDHSA) return legalizeTrapEndpgm(MI, MRI, B); @@ -6726,8 +6732,9 @@ bool AMDGPULegalizerInfo::legalizeTrapHsa( return true; } -bool AMDGPULegalizerInfo::legalizeDebugTrapIntrinsic( - MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const { +bool AMDGPULegalizerInfo::legalizeDebugTrap(MachineInstr &MI, + MachineRegisterInfo &MRI, + MachineIRBuilder &B) const { // Is non-HSA path or trap-handler disabled? Then, report a warning // accordingly if (!ST.isTrapHandlerEnabled() || @@ -7270,10 +7277,6 @@ bool AMDGPULegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper, case Intrinsic::amdgcn_struct_buffer_atomic_fadd_v2bf16: case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fadd_v2bf16: return legalizeBufferAtomic(MI, B, IntrID); - case Intrinsic::trap: - return legalizeTrapIntrinsic(MI, MRI, B); - case Intrinsic::debugtrap: - return legalizeDebugTrapIntrinsic(MI, MRI, B); case Intrinsic::amdgcn_rsq_clamp: return legalizeRsqClampIntrinsic(MI, MRI, B); case Intrinsic::amdgcn_ds_fadd: diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h index 9661646fffc9..e5ba84a74a0f 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h +++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h @@ -226,16 +226,16 @@ public: bool legalizeSBufferLoad(LegalizerHelper &Helper, MachineInstr &MI) const; - bool legalizeTrapIntrinsic(MachineInstr &MI, MachineRegisterInfo &MRI, - MachineIRBuilder &B) const; + bool legalizeTrap(MachineInstr &MI, MachineRegisterInfo &MRI, + MachineIRBuilder &B) const; bool legalizeTrapEndpgm(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const; bool legalizeTrapHsaQueuePtr(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const; bool legalizeTrapHsa(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const; - bool legalizeDebugTrapIntrinsic(MachineInstr &MI, MachineRegisterInfo &MRI, - MachineIRBuilder &B) const; + bool legalizeDebugTrap(MachineInstr &MI, MachineRegisterInfo &MRI, + MachineIRBuilder &B) const; bool legalizeIntrinsic(LegalizerHelper &Helper, MachineInstr &MI) const override; diff --git a/llvm/lib/Target/Mips/MipsLegalizerInfo.cpp b/llvm/lib/Target/Mips/MipsLegalizerInfo.cpp index 3307e840a2af..8468dd6a2211 100644 --- a/llvm/lib/Target/Mips/MipsLegalizerInfo.cpp +++ b/llvm/lib/Target/Mips/MipsLegalizerInfo.cpp @@ -508,16 +508,8 @@ bool MipsLegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper, MachineInstr &MI) const { MachineIRBuilder &MIRBuilder = Helper.MIRBuilder; const MipsSubtarget &ST = MI.getMF()->getSubtarget(); - const MipsInstrInfo &TII = *ST.getInstrInfo(); - const MipsRegisterInfo &TRI = *ST.getRegisterInfo(); - const RegisterBankInfo &RBI = *ST.getRegBankInfo(); switch (cast(MI).getIntrinsicID()) { - case Intrinsic::trap: { - MachineInstr *Trap = MIRBuilder.buildInstr(Mips::TRAP); - MI.eraseFromParent(); - return constrainSelectedInstRegOperands(*Trap, TII, TRI, RBI); - } case Intrinsic::vacopy: { MachinePointerInfo MPO; LLT PtrTy = LLT::pointer(0, 32); diff --git a/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp b/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp index 5738f86e7e9f..3103992a86c0 100644 --- a/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp +++ b/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp @@ -77,8 +77,6 @@ private: MachineRegisterInfo &MRI) const; bool selectFPCompare(MachineInstr &MI, MachineIRBuilder &MIB, MachineRegisterInfo &MRI) const; - bool selectIntrinsicWithSideEffects(MachineInstr &MI, MachineIRBuilder &MIB, - MachineRegisterInfo &MRI) const; void emitFence(AtomicOrdering FenceOrdering, SyncScope::ID FenceSSID, MachineIRBuilder &MIB) const; bool selectMergeValues(MachineInstr &MI, MachineIRBuilder &MIB, @@ -686,8 +684,6 @@ bool RISCVInstructionSelector::select(MachineInstr &MI) { return selectSelect(MI, MIB, MRI); case TargetOpcode::G_FCMP: return selectFPCompare(MI, MIB, MRI); - case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: - return selectIntrinsicWithSideEffects(MI, MIB, MRI); case TargetOpcode::G_FENCE: { AtomicOrdering FenceOrdering = static_cast(MI.getOperand(0).getImm()); @@ -1255,29 +1251,6 @@ bool RISCVInstructionSelector::selectFPCompare(MachineInstr &MI, return true; } -bool RISCVInstructionSelector::selectIntrinsicWithSideEffects( - MachineInstr &MI, MachineIRBuilder &MIB, MachineRegisterInfo &MRI) const { - assert(MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS && - "Unexpected opcode"); - // Find the intrinsic ID. - unsigned IntrinID = cast(MI).getIntrinsicID(); - - // Select the instruction. - switch (IntrinID) { - default: - return false; - case Intrinsic::trap: - MIB.buildInstr(RISCV::UNIMP, {}, {}); - break; - case Intrinsic::debugtrap: - MIB.buildInstr(RISCV::EBREAK, {}, {}); - break; - } - - MI.eraseFromParent(); - return true; -} - void RISCVInstructionSelector::emitFence(AtomicOrdering FenceOrdering, SyncScope::ID FenceSSID, MachineIRBuilder &MIB) const { diff --git a/llvm/lib/Target/X86/GISel/X86InstructionSelector.cpp b/llvm/lib/Target/X86/GISel/X86InstructionSelector.cpp index 8e0f61a85566..9be3812300af 100644 --- a/llvm/lib/Target/X86/GISel/X86InstructionSelector.cpp +++ b/llvm/lib/Target/X86/GISel/X86InstructionSelector.cpp @@ -119,8 +119,6 @@ private: MachineFunction &MF) const; bool selectSelect(MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF) const; - bool selectIntrinsicWSideEffects(MachineInstr &I, MachineRegisterInfo &MRI, - MachineFunction &MF) const; // emit insert subreg instruction and insert it before MachineInstr &I bool emitInsertSubreg(unsigned DstReg, unsigned SrcReg, MachineInstr &I, @@ -434,8 +432,6 @@ bool X86InstructionSelector::select(MachineInstr &I) { return selectMulDivRem(I, MRI, MF); case TargetOpcode::G_SELECT: return selectSelect(I, MRI, MF); - case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: - return selectIntrinsicWSideEffects(I, MRI, MF); } return false; @@ -1834,21 +1830,6 @@ bool X86InstructionSelector::selectSelect(MachineInstr &I, return true; } -bool X86InstructionSelector::selectIntrinsicWSideEffects( - MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF) const { - - assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS && - "unexpected instruction"); - - if (I.getOperand(0).getIntrinsicID() != Intrinsic::trap) - return false; - - BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::TRAP)); - - I.eraseFromParent(); - return true; -} - InstructionSelector * llvm::createX86InstructionSelector(const X86TargetMachine &TM, X86Subtarget &Subtarget, diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-unreachable.ll b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-unreachable.ll index fe9427d2678a..edae903fae84 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-unreachable.ll +++ b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-unreachable.ll @@ -6,7 +6,7 @@ declare void @llvm.trap() define void @unreachable() { ; CHECK-LABEL: name: unreachable ; CHECK: bb.1 (%ir-block.0): - ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + ; CHECK-NEXT: G_TRAP unreachable ret void } diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-exceptions.ll b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-exceptions.ll index 5662de4cbdca..f7550ceb2799 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-exceptions.ll +++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-exceptions.ll @@ -48,7 +48,7 @@ define void @bar() personality ptr @__gxx_personality_v0 { ; CHECK-NEXT: $x0 = COPY [[LOAD]](p0) ; CHECK-NEXT: BL @_Unwind_Resume, csr_darwin_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $x0 ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp - ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + ; CHECK-NEXT: G_TRAP %exn.slot = alloca ptr %ehselector.slot = alloca i32 %1 = invoke i32 @foo(i32 42) to label %continue unwind label %cleanup diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir index c9e5f8924f8a..ac3c47c8001d 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir @@ -752,6 +752,15 @@ # DEBUG-NEXT: G_BZERO (opcode {{[0-9]+}}): 2 type indices, 1 imm index # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected # DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected +# DEBUG-NEXT: G_TRAP (opcode {{[0-9]+}}): 0 type indices, 0 imm indices +# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined +# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined +# DEBUG-NEXT: G_DEBUGTRAP (opcode {{[0-9]+}}): 0 type indices, 0 imm indices +# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined +# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined +# DEBUG-NEXT: G_UBSANTRAP (opcode {{[0-9]+}}): 0 type indices, 0 imm indices +# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined +# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined # DEBUG-NEXT: G_VECREDUCE_SEQ_FADD (opcode {{[0-9]+}}): 3 type indices, 0 imm indices # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected # DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/select-trap.mir b/llvm/test/CodeGen/AArch64/GlobalISel/select-trap.mir index ad66fa5623e3..25ecce4dd92b 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/select-trap.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/select-trap.mir @@ -26,7 +26,7 @@ body: | ; CHECK-LABEL: name: foo ; CHECK: BRK 1 ; CHECK: RET_ReallyLR - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP RET_ReallyLR ... diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/uaddo-8-16-bits.mir b/llvm/test/CodeGen/AArch64/GlobalISel/uaddo-8-16-bits.mir index f4366fb7888e..b242c68e3b07 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/uaddo-8-16-bits.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/uaddo-8-16-bits.mir @@ -26,7 +26,7 @@ body: | ; CHECK-NEXT: bb.1: ; CHECK-NEXT: successors: ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + ; CHECK-NEXT: G_TRAP ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.2: ; CHECK-NEXT: $w0 = COPY [[ADD]](s32) @@ -48,7 +48,7 @@ body: | bb.2: successors: - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP bb.3: %8:_(s32) = G_ZEXT %6(s8) @@ -80,7 +80,7 @@ body: | ; CHECK-NEXT: bb.1: ; CHECK-NEXT: successors: ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + ; CHECK-NEXT: G_TRAP ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.2: ; CHECK-NEXT: $w0 = COPY [[ADD]](s32) @@ -102,7 +102,7 @@ body: | bb.2: successors: - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP bb.3: %8:_(s32) = G_ZEXT %6(s16) @@ -134,7 +134,7 @@ body: | ; CHECK-NEXT: bb.1: ; CHECK-NEXT: successors: ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + ; CHECK-NEXT: G_TRAP ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.2: ; CHECK-NEXT: liveins: $x2 @@ -165,7 +165,7 @@ body: | bb.2: successors: - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP bb.3: liveins: $x2 @@ -206,7 +206,7 @@ body: | ; CHECK-NEXT: bb.1: ; CHECK-NEXT: successors: ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + ; CHECK-NEXT: G_TRAP ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.2: ; CHECK-NEXT: $w0 = COPY [[ADD]](s32) @@ -228,7 +228,7 @@ body: | bb.2: successors: - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP bb.3: %8:_(s32) = G_ANYEXT %6(s16) @@ -261,7 +261,7 @@ body: | ; CHECK-NEXT: bb.1: ; CHECK-NEXT: successors: ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + ; CHECK-NEXT: G_TRAP ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.2: ; CHECK-NEXT: $w0 = COPY [[ADD]](s32) @@ -284,7 +284,7 @@ body: | bb.2: successors: - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP bb.3: %8:_(s32) = G_ZEXT %6(s16) @@ -317,7 +317,7 @@ body: | ; CHECK-NEXT: bb.1: ; CHECK-NEXT: successors: ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + ; CHECK-NEXT: G_TRAP ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.2: ; CHECK-NEXT: $w0 = COPY [[ADD]](s32) @@ -340,7 +340,7 @@ body: | bb.2: successors: - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP bb.3: %8:_(s32) = G_ZEXT %6(s16) @@ -377,7 +377,7 @@ body: | ; CHECK-NEXT: bb.1: ; CHECK-NEXT: successors: ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + ; CHECK-NEXT: G_TRAP ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.2: ; CHECK-NEXT: liveins: $x2 @@ -410,7 +410,7 @@ body: | bb.2: successors: - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP bb.3: liveins: $x2 @@ -512,7 +512,7 @@ body: | ; CHECK-NEXT: bb.2: ; CHECK-NEXT: successors: ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + ; CHECK-NEXT: G_TRAP ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.3: ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO]](s16) @@ -544,7 +544,7 @@ body: | bb.2: successors: - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP bb.3: %9:_(s32) = G_ZEXT %6(s16) @@ -577,7 +577,7 @@ body: | ; CHECK-NEXT: bb.1: ; CHECK-NEXT: successors: ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + ; CHECK-NEXT: G_TRAP ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.2: ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO]](s8) @@ -601,7 +601,7 @@ body: | bb.2: successors: - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP bb.3: %9:_(s32) = G_ZEXT %7(s8) @@ -634,7 +634,7 @@ body: | ; CHECK-NEXT: bb.1: ; CHECK-NEXT: successors: ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + ; CHECK-NEXT: G_TRAP ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.2: ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO]](s8) @@ -658,7 +658,7 @@ body: | bb.2: successors: - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP bb.3: %9:_(s32) = G_ZEXT %7(s8) @@ -692,7 +692,7 @@ body: | ; CHECK-NEXT: bb.1: ; CHECK-NEXT: successors: ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + ; CHECK-NEXT: G_TRAP ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.2: ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO]](s8) @@ -717,7 +717,7 @@ body: | bb.2: successors: - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP bb.3: %10:_(s32) = G_ZEXT %8(s8) @@ -783,7 +783,7 @@ body: | ; CHECK-NEXT: bb.1: ; CHECK-NEXT: successors: ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + ; CHECK-NEXT: G_TRAP ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.2: ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UADDO]](s16) @@ -804,7 +804,7 @@ body: | bb.2: successors: - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP bb.3: %6:_(s32) = G_ANYEXT %4(s16) @@ -839,7 +839,7 @@ body: | ; CHECK-NEXT: RET_ReallyLR implicit $w0 ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.2: - ; CHECK-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + ; CHECK-NEXT: G_TRAP bb.1: successors: %bb.2(0x7ffff800), %bb.3(0x00000800) liveins: $w0, $w1 @@ -860,6 +860,6 @@ body: | RET_ReallyLR implicit $w0 bb.3: - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP ... diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-trap.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-trap.mir index b4bc64812b53..305eca792cfb 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-trap.mir +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-trap.mir @@ -24,7 +24,7 @@ body: | bb.0: %0:_(s8) = G_CONSTANT i8 0 %1:_(p1) = G_CONSTANT i64 0 - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP bb.1: G_STORE %0, %1 :: (store 1, addrspace 1) @@ -55,7 +55,7 @@ body: | ; GCN-NEXT: S_ENDPGM 0 bb.0: %0:_(s8) = G_CONSTANT i8 0 - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP %1:_(p1) = G_CONSTANT i64 0 bb.1: diff --git a/llvm/test/CodeGen/Mips/GlobalISel/legalizer/trap.mir b/llvm/test/CodeGen/Mips/GlobalISel/instruction-select/trap.mir similarity index 55% rename from llvm/test/CodeGen/Mips/GlobalISel/legalizer/trap.mir rename to llvm/test/CodeGen/Mips/GlobalISel/instruction-select/trap.mir index 64388933fda8..dc99ce8d7a09 100644 --- a/llvm/test/CodeGen/Mips/GlobalISel/legalizer/trap.mir +++ b/llvm/test/CodeGen/Mips/GlobalISel/instruction-select/trap.mir @@ -1,5 +1,5 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py -# RUN: llc -O0 -mtriple=mipsel-linux-gnu -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s -check-prefixes=MIPS32 +# RUN: llc -O0 -mtriple=mipsel-linux-gnu -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s -check-prefixes=MIPS32 --- | declare void @llvm.trap() @@ -9,12 +9,15 @@ --- name: f alignment: 4 +legalized: true +regBankSelected: true +tracksRegLiveness: true body: | bb.1 (%ir-block.0): ; MIPS32-LABEL: name: f ; MIPS32: TRAP - ; MIPS32: RetRA - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + ; MIPS32-NEXT: RetRA + G_TRAP RetRA ... diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/trap.mir b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/trap.mir index 11789a030e6f..5f52030fc170 100644 --- a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/trap.mir +++ b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/trap.mir @@ -14,7 +14,7 @@ body: | ; CHECK-LABEL: name: test_trap ; CHECK: UNIMP ; CHECK-NEXT: PseudoRET - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP PseudoRET ... @@ -28,7 +28,7 @@ body: | ; CHECK-LABEL: name: test_debugtrap ; CHECK: EBREAK ; CHECK-NEXT: PseudoRET - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.debugtrap) + G_DEBUGTRAP PseudoRET ... diff --git a/llvm/test/CodeGen/X86/GlobalISel/x86-select-trap.mir b/llvm/test/CodeGen/X86/GlobalISel/x86-select-trap.mir index ea548c296dca..20b8b671ac5a 100644 --- a/llvm/test/CodeGen/X86/GlobalISel/x86-select-trap.mir +++ b/llvm/test/CodeGen/X86/GlobalISel/x86-select-trap.mir @@ -23,6 +23,6 @@ body: | bb.1 (%ir-block.0): ; CHECK-LABEL: name: trap ; CHECK: TRAP - G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap) + G_TRAP ... diff --git a/llvm/test/CodeGen/X86/isel-traps.ll b/llvm/test/CodeGen/X86/isel-traps.ll new file mode 100644 index 000000000000..c207387166a6 --- /dev/null +++ b/llvm/test/CodeGen/X86/isel-traps.ll @@ -0,0 +1,73 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-linux-gnu | FileCheck %s --check-prefixes=ALL,X64 +; RUN: llc < %s -fast-isel -fast-isel-abort=1 -mtriple=x86_64-linux-gnu | FileCheck %s --check-prefixes=ALL,X64 +; RUN: llc < %s -global-isel -global-isel-abort=1 -mtriple=x86_64-linux-gnu | FileCheck %s --check-prefixes=ALL,GISEL-X64 +; RUN: llc < %s -mtriple=i686-linux-gnu | FileCheck %s --check-prefixes=ALL,X86 +; RUN: llc < %s -fast-isel -fast-isel-abort=1 -mtriple=i686-linux-gnu | FileCheck %s --check-prefixes=ALL,X86 +; RUN: llc < %s -global-isel -global-isel-abort=1 -mtriple=i686-linux-gnu | FileCheck %s --check-prefixes=ALL,GISEL-X86 + +declare void @llvm.trap() + +define void @test_trap() { +; ALL-LABEL: test_trap: +; ALL: # %bb.0: +; ALL-NEXT: ud2 +; ALL-NEXT: ret{{[l|q]}} + tail call void @llvm.trap() + ret void +} + +define void @test_debugtrap() { +; ALL-LABEL: test_debugtrap: +; ALL: # %bb.0: +; ALL-NEXT: int3 +; ALL-NEXT: ret{{[l|q]}} + tail call void @llvm.debugtrap() + ret void +} + +define void @test_ubsantrap() { +; ALL-LABEL: test_ubsantrap: +; ALL: # %bb.0: +; ALL-NEXT: ud1l 12(%eax), %eax +; ALL-NEXT: ret{{[l|q]}} + call void @llvm.ubsantrap(i8 12) + ret void +} + +define void @test_ubsantrap_custom() nounwind { +; X64-LABEL: test_ubsantrap_custom: +; X64: # %bb.0: +; X64-NEXT: pushq %rax +; X64-NEXT: movl $42, %edi +; X64-NEXT: callq guide@PLT +; X64-NEXT: popq %rax +; X64-NEXT: retq +; +; GISEL-X64-LABEL: test_ubsantrap_custom: +; GISEL-X64: # %bb.0: +; GISEL-X64-NEXT: pushq %rax +; GISEL-X64-NEXT: movl $42, %edi +; GISEL-X64-NEXT: callq guide +; GISEL-X64-NEXT: popq %rax +; GISEL-X64-NEXT: retq +; +; X86-LABEL: test_ubsantrap_custom: +; X86: # %bb.0: +; X86-NEXT: subl $12, %esp +; X86-NEXT: movl $42, (%esp) +; X86-NEXT: calll guide +; X86-NEXT: addl $12, %esp +; X86-NEXT: retl +; +; GISEL-X86-LABEL: test_ubsantrap_custom: +; GISEL-X86: # %bb.0: +; GISEL-X86-NEXT: subl $12, %esp +; GISEL-X86-NEXT: movl $42, %eax +; GISEL-X86-NEXT: movl %eax, (%esp) +; GISEL-X86-NEXT: calll guide +; GISEL-X86-NEXT: addl $12, %esp +; GISEL-X86-NEXT: retl + call void @llvm.ubsantrap(i8 42) "trap-func-name"="guide" + ret void +} diff --git a/llvm/test/MachineVerifier/test_g_ubsantrap.mir b/llvm/test/MachineVerifier/test_g_ubsantrap.mir new file mode 100644 index 000000000000..d2b219d8650a --- /dev/null +++ b/llvm/test/MachineVerifier/test_g_ubsantrap.mir @@ -0,0 +1,18 @@ +# RUN: not --crash llc -o - -mtriple=arm64 -run-pass=none -verify-machineinstrs %s 2>&1 | FileCheck %s +# REQUIRES: aarch64-registered-target + +--- +name: test_ubsantrap +tracksRegLiveness: true +liveins: +body: | + bb.0: + + ; CHECK: Crash kind must be 8 bit wide + G_UBSANTRAP 4096 + + ; CHECK: Crash kind must be an immediate + %5:_(s32) = IMPLICIT_DEF + G_UBSANTRAP %5 + +... -- GitLab From f886dfed3ae6cf70827cedc8d8aefde6250a239b Mon Sep 17 00:00:00 2001 From: Nikolas Klauser Date: Sat, 23 Mar 2024 13:54:35 +0100 Subject: [PATCH 144/306] [libc++] Don't push and pop extensions diagnostics when using clang modules (#85917) Clang modules take a significant compile time hit when pushing and popping diagnostics. Since all the headers are marked as system headers in the modulemap, we can simply disable this pushing and popping when building with clang modules. --- libcxx/include/__config | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/libcxx/include/__config b/libcxx/include/__config index 132cace3d5b2..af44af203e7c 100644 --- a/libcxx/include/__config +++ b/libcxx/include/__config @@ -838,21 +838,33 @@ typedef __char32_t char32_t; # define _LIBCPP_CLANG_DIAGNOSTIC_IGNORED_CXX23_EXTENSION _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++2b-extensions") #endif +// Clang modules take a significant compile time hit when pushing and popping diagnostics. +// Since all the headers are marked as system headers in the modulemap, we can simply disable this +// pushing and popping when building with clang modules. +# if !__has_feature(modules) +# define _LIBCPP_PUSH_EXTENSION_DIAGNOSTICS \ + _LIBCPP_DIAGNOSTIC_PUSH \ + _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++11-extensions") \ + _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++14-extensions") \ + _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++17-extensions") \ + _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++20-extensions") \ + _LIBCPP_CLANG_DIAGNOSTIC_IGNORED_CXX23_EXTENSION \ + _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wc++14-extensions") \ + _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wc++17-extensions") \ + _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wc++20-extensions") \ + _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wc++23-extensions") +# define _LIBCPP_POP_EXTENSION_DIAGNOSTICS _LIBCPP_DIAGNOSTIC_POP +# else +# define _LIBCPP_PUSH_EXTENSION_DIAGNOSTICS +# define _LIBCPP_POP_EXTENSION_DIAGNOSTICS +# endif + // Inline namespaces are available in Clang/GCC/MSVC regardless of C++ dialect. // clang-format off -# define _LIBCPP_BEGIN_NAMESPACE_STD _LIBCPP_DIAGNOSTIC_PUSH \ - _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++11-extensions") \ - _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++14-extensions") \ - _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++17-extensions") \ - _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++20-extensions") \ - _LIBCPP_CLANG_DIAGNOSTIC_IGNORED_CXX23_EXTENSION \ - _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wc++14-extensions") \ - _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wc++17-extensions") \ - _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wc++20-extensions") \ - _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wc++23-extensions") \ +# define _LIBCPP_BEGIN_NAMESPACE_STD _LIBCPP_PUSH_EXTENSION_DIAGNOSTICS \ namespace _LIBCPP_TYPE_VISIBILITY_DEFAULT std { \ inline namespace _LIBCPP_ABI_NAMESPACE { -# define _LIBCPP_END_NAMESPACE_STD }} _LIBCPP_DIAGNOSTIC_POP +# define _LIBCPP_END_NAMESPACE_STD }} _LIBCPP_POP_EXTENSION_DIAGNOSTICS # define _LIBCPP_BEGIN_NAMESPACE_FILESYSTEM _LIBCPP_BEGIN_NAMESPACE_STD \ inline namespace __fs { namespace filesystem { -- GitLab From 57146daeaaf366050dc913db910fcc2995a3e06d Mon Sep 17 00:00:00 2001 From: Harvin Iriawan <25712785+harviniriawan@users.noreply.github.com> Date: Sat, 23 Mar 2024 12:56:25 +0000 Subject: [PATCH 145/306] [CodeGen] Update for scalable MemoryType in MMO (#70452) Remove getSizeOrUnknown call when MachineMemOperand is created. For Scalable TypeSize, the MemoryType created becomes a scalable_vector. 2 MMOs that have scalable memory access can then use the updated BasicAA that understands scalable LocationSize. Original Patch by Harvin Iriawan Co-authored-by: David Green --- llvm/include/llvm/Analysis/MemoryLocation.h | 7 --- llvm/include/llvm/CodeGen/MachineFunction.h | 5 +- llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp | 34 +++++++++---- llvm/lib/CodeGen/MachineInstr.cpp | 37 ++++++++++---- llvm/lib/CodeGen/MachineOperand.cpp | 13 ++--- llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 51 ++++++++++++------- .../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 22 ++++---- .../SelectionDAGAddressAnalysis.cpp | 2 - .../SelectionDAG/SelectionDAGBuilder.cpp | 3 +- llvm/lib/Target/AArch64/AArch64InstrInfo.cpp | 5 +- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 22 ++++++-- llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll | 6 +-- .../alloca-load-store-scalable-array.ll | 36 ++++++------- .../alloca-load-store-scalable-struct.ll | 12 ++--- .../rvv/alloca-load-store-scalable-array.ll | 12 ++--- .../rvv/alloca-load-store-scalable-struct.ll | 8 +-- .../RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll | 8 +-- 17 files changed, 165 insertions(+), 118 deletions(-) diff --git a/llvm/include/llvm/Analysis/MemoryLocation.h b/llvm/include/llvm/Analysis/MemoryLocation.h index 830eed5d60ee..7d896c44f467 100644 --- a/llvm/include/llvm/Analysis/MemoryLocation.h +++ b/llvm/include/llvm/Analysis/MemoryLocation.h @@ -297,13 +297,6 @@ public: return MemoryLocation(Ptr, LocationSize::beforeOrAfterPointer(), AATags); } - // Return the exact size if the exact size is known at compiletime, - // otherwise return LocationSize::beforeOrAfterPointer(). - static LocationSize getSizeOrUnknown(const TypeSize &T) { - return T.isScalable() ? LocationSize::beforeOrAfterPointer() - : LocationSize::precise(T.getFixedValue()); - } - MemoryLocation() : Ptr(nullptr), Size(LocationSize::beforeOrAfterPointer()) {} explicit MemoryLocation(const Value *Ptr, LocationSize Size, diff --git a/llvm/include/llvm/CodeGen/MachineFunction.h b/llvm/include/llvm/CodeGen/MachineFunction.h index dfbf7a1e7aae..c2bff2794493 100644 --- a/llvm/include/llvm/CodeGen/MachineFunction.h +++ b/llvm/include/llvm/CodeGen/MachineFunction.h @@ -1058,8 +1058,9 @@ public: int64_t Offset, LocationSize Size) { return getMachineMemOperand( MMO, Offset, - !Size.hasValue() || Size.isScalable() - ? LLT() + !Size.hasValue() ? LLT() + : Size.isScalable() + ? LLT::scalable_vector(1, 8 * Size.getValue().getKnownMinValue()) : LLT::scalar(8 * Size.getValue().getKnownMinValue())); } MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO, diff --git a/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp b/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp index 9fc8ecd60b03..fb9656c09ca3 100644 --- a/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp +++ b/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp @@ -128,14 +128,14 @@ bool GISelAddressing::aliasIsKnownForLoadStore(const MachineInstr &MI1, // vector objects on the stack. // BasePtr1 is PtrDiff away from BasePtr0. They alias if none of the // following situations arise: - if (PtrDiff >= 0 && Size1.hasValue()) { + if (PtrDiff >= 0 && Size1.hasValue() && !Size1.isScalable()) { // [----BasePtr0----] // [---BasePtr1--] // ========PtrDiff========> IsAlias = !((int64_t)Size1.getValue() <= PtrDiff); return true; } - if (PtrDiff < 0 && Size2.hasValue()) { + if (PtrDiff < 0 && Size2.hasValue() && !Size2.isScalable()) { // [----BasePtr0----] // [---BasePtr1--] // =====(-PtrDiff)====> @@ -248,10 +248,20 @@ bool GISelAddressing::instMayAlias(const MachineInstr &MI, return false; } + // If NumBytes is scalable and offset is not 0, conservatively return may + // alias + if ((MUC0.NumBytes.isScalable() && MUC0.Offset != 0) || + (MUC1.NumBytes.isScalable() && MUC1.Offset != 0)) + return true; + + const bool BothNotScalable = + !MUC0.NumBytes.isScalable() && !MUC1.NumBytes.isScalable(); + // Try to prove that there is aliasing, or that there is no aliasing. Either // way, we can return now. If nothing can be proved, proceed with more tests. bool IsAlias; - if (GISelAddressing::aliasIsKnownForLoadStore(MI, Other, IsAlias, MRI)) + if (BothNotScalable && + GISelAddressing::aliasIsKnownForLoadStore(MI, Other, IsAlias, MRI)) return IsAlias; // The following all rely on MMO0 and MMO1 being valid. @@ -267,12 +277,18 @@ bool GISelAddressing::instMayAlias(const MachineInstr &MI, Size1.hasValue()) { // Use alias analysis information. int64_t MinOffset = std::min(SrcValOffset0, SrcValOffset1); - int64_t Overlap0 = Size0.getValue() + SrcValOffset0 - MinOffset; - int64_t Overlap1 = Size1.getValue() + SrcValOffset1 - MinOffset; - if (AA->isNoAlias(MemoryLocation(MUC0.MMO->getValue(), Overlap0, - MUC0.MMO->getAAInfo()), - MemoryLocation(MUC1.MMO->getValue(), Overlap1, - MUC1.MMO->getAAInfo()))) + int64_t Overlap0 = + Size0.getValue().getKnownMinValue() + SrcValOffset0 - MinOffset; + int64_t Overlap1 = + Size1.getValue().getKnownMinValue() + SrcValOffset1 - MinOffset; + LocationSize Loc0 = + Size0.isScalable() ? Size0 : LocationSize::precise(Overlap0); + LocationSize Loc1 = + Size1.isScalable() ? Size1 : LocationSize::precise(Overlap1); + + if (AA->isNoAlias( + MemoryLocation(MUC0.MMO->getValue(), Loc0, MUC0.MMO->getAAInfo()), + MemoryLocation(MUC1.MMO->getValue(), Loc1, MUC1.MMO->getAAInfo()))) return false; } diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp index fe2f9ccd33a3..8102bb971ba6 100644 --- a/llvm/lib/CodeGen/MachineInstr.cpp +++ b/llvm/lib/CodeGen/MachineInstr.cpp @@ -1306,6 +1306,7 @@ static bool MemOperandsHaveAlias(const MachineFrameInfo &MFI, AAResults *AA, LocationSize WidthB = MMOb->getSize(); bool KnownWidthA = WidthA.hasValue(); bool KnownWidthB = WidthB.hasValue(); + bool BothMMONonScalable = !WidthA.isScalable() && !WidthB.isScalable(); const Value *ValA = MMOa->getValue(); const Value *ValB = MMOb->getValue(); @@ -1321,12 +1322,14 @@ static bool MemOperandsHaveAlias(const MachineFrameInfo &MFI, AAResults *AA, SameVal = true; } - if (SameVal) { + if (SameVal && BothMMONonScalable) { if (!KnownWidthA || !KnownWidthB) return true; int64_t MaxOffset = std::max(OffsetA, OffsetB); - LocationSize LowWidth = (MinOffset == OffsetA) ? WidthA : WidthB; - return (MinOffset + (int)LowWidth.getValue() > MaxOffset); + int64_t LowWidth = (MinOffset == OffsetA) + ? WidthA.getValue().getKnownMinValue() + : WidthB.getValue().getKnownMinValue(); + return (MinOffset + LowWidth > MaxOffset); } if (!AA) @@ -1338,15 +1341,29 @@ static bool MemOperandsHaveAlias(const MachineFrameInfo &MFI, AAResults *AA, assert((OffsetA >= 0) && "Negative MachineMemOperand offset"); assert((OffsetB >= 0) && "Negative MachineMemOperand offset"); - int64_t OverlapA = KnownWidthA ? WidthA.getValue() + OffsetA - MinOffset - : MemoryLocation::UnknownSize; - int64_t OverlapB = KnownWidthB ? WidthB.getValue() + OffsetB - MinOffset - : MemoryLocation::UnknownSize; + // If Scalable Location Size has non-zero offset, Width + Offset does not work + // at the moment + if ((WidthA.isScalable() && OffsetA > 0) || + (WidthB.isScalable() && OffsetB > 0)) + return true; + + int64_t OverlapA = + KnownWidthA ? WidthA.getValue().getKnownMinValue() + OffsetA - MinOffset + : MemoryLocation::UnknownSize; + int64_t OverlapB = + KnownWidthB ? WidthB.getValue().getKnownMinValue() + OffsetB - MinOffset + : MemoryLocation::UnknownSize; + + LocationSize LocA = (WidthA.isScalable() || !KnownWidthA) + ? WidthA + : LocationSize::precise(OverlapA); + LocationSize LocB = (WidthB.isScalable() || !KnownWidthB) + ? WidthB + : LocationSize::precise(OverlapB); return !AA->isNoAlias( - MemoryLocation(ValA, OverlapA, UseTBAA ? MMOa->getAAInfo() : AAMDNodes()), - MemoryLocation(ValB, OverlapB, - UseTBAA ? MMOb->getAAInfo() : AAMDNodes())); + MemoryLocation(ValA, LocA, UseTBAA ? MMOa->getAAInfo() : AAMDNodes()), + MemoryLocation(ValB, LocB, UseTBAA ? MMOb->getAAInfo() : AAMDNodes())); } bool MachineInstr::mayAlias(AAResults *AA, const MachineInstr &Other, diff --git a/llvm/lib/CodeGen/MachineOperand.cpp b/llvm/lib/CodeGen/MachineOperand.cpp index 937ca539513a..ace05902d5df 100644 --- a/llvm/lib/CodeGen/MachineOperand.cpp +++ b/llvm/lib/CodeGen/MachineOperand.cpp @@ -1107,12 +1107,13 @@ MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags F, const MDNode *Ranges, SyncScope::ID SSID, AtomicOrdering Ordering, AtomicOrdering FailureOrdering) - : MachineMemOperand(ptrinfo, F, - !TS.hasValue() || TS.isScalable() - ? LLT() - : LLT::scalar(8 * TS.getValue().getKnownMinValue()), - BaseAlignment, AAInfo, Ranges, SSID, Ordering, - FailureOrdering) {} + : MachineMemOperand( + ptrinfo, F, + !TS.hasValue() ? LLT() + : TS.isScalable() + ? LLT::scalable_vector(1, 8 * TS.getValue().getKnownMinValue()) + : LLT::scalar(8 * TS.getValue().getKnownMinValue()), + BaseAlignment, AAInfo, Ranges, SSID, Ordering, FailureOrdering) {} void MachineMemOperand::refineAlignment(const MachineMemOperand *MMO) { // The Value and Offset may differ due to CSE. But the flags and size diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index dcd0310734ad..e27a8bb8fdac 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -24200,7 +24200,7 @@ static SDValue narrowExtractedVectorLoad(SDNode *Extract, SelectionDAG &DAG) { // TODO: Use "BaseIndexOffset" to make this more effective. SDValue NewAddr = DAG.getMemBasePlusOffset(Ld->getBasePtr(), Offset, DL); - LocationSize StoreSize = MemoryLocation::getSizeOrUnknown(VT.getStoreSize()); + LocationSize StoreSize = LocationSize::precise(VT.getStoreSize()); MachineFunction &MF = DAG.getMachineFunction(); MachineMemOperand *MMO; if (Offset.isScalable()) { @@ -27845,14 +27845,10 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const { : (LSN->getAddressingMode() == ISD::PRE_DEC) ? -1 * C->getSExtValue() : 0; - LocationSize Size = - MemoryLocation::getSizeOrUnknown(LSN->getMemoryVT().getStoreSize()); - return {LSN->isVolatile(), - LSN->isAtomic(), - LSN->getBasePtr(), - Offset /*base offset*/, - Size, - LSN->getMemOperand()}; + TypeSize Size = LSN->getMemoryVT().getStoreSize(); + return {LSN->isVolatile(), LSN->isAtomic(), + LSN->getBasePtr(), Offset /*base offset*/, + LocationSize::precise(Size), LSN->getMemOperand()}; } if (const auto *LN = cast(N)) return {false /*isVolatile*/, @@ -27894,6 +27890,13 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const { return false; } + // If NumBytes is scalable and offset is not 0, conservatively return may + // alias + if ((MUC0.NumBytes.hasValue() && MUC0.NumBytes.isScalable() && + MUC0.Offset != 0) || + (MUC1.NumBytes.hasValue() && MUC1.NumBytes.isScalable() && + MUC1.Offset != 0)) + return true; // Try to prove that there is aliasing, or that there is no aliasing. Either // way, we can return now. If nothing can be proved, proceed with more tests. bool IsAlias; @@ -27924,18 +27927,22 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const { Align OrigAlignment1 = MUC1.MMO->getBaseAlign(); LocationSize Size0 = MUC0.NumBytes; LocationSize Size1 = MUC1.NumBytes; + if (OrigAlignment0 == OrigAlignment1 && SrcValOffset0 != SrcValOffset1 && - Size0.hasValue() && Size1.hasValue() && Size0 == Size1 && - OrigAlignment0 > Size0.getValue() && - SrcValOffset0 % Size0.getValue() == 0 && - SrcValOffset1 % Size1.getValue() == 0) { + Size0.hasValue() && Size1.hasValue() && !Size0.isScalable() && + !Size1.isScalable() && Size0 == Size1 && + OrigAlignment0 > Size0.getValue().getKnownMinValue() && + SrcValOffset0 % Size0.getValue().getKnownMinValue() == 0 && + SrcValOffset1 % Size1.getValue().getKnownMinValue() == 0) { int64_t OffAlign0 = SrcValOffset0 % OrigAlignment0.value(); int64_t OffAlign1 = SrcValOffset1 % OrigAlignment1.value(); // There is no overlap between these relatively aligned accesses of // similar size. Return no alias. - if ((OffAlign0 + (int64_t)Size0.getValue()) <= OffAlign1 || - (OffAlign1 + (int64_t)Size1.getValue()) <= OffAlign0) + if ((OffAlign0 + static_cast( + Size0.getValue().getKnownMinValue())) <= OffAlign1 || + (OffAlign1 + static_cast( + Size1.getValue().getKnownMinValue())) <= OffAlign0) return false; } @@ -27952,12 +27959,18 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const { Size0.hasValue() && Size1.hasValue()) { // Use alias analysis information. int64_t MinOffset = std::min(SrcValOffset0, SrcValOffset1); - int64_t Overlap0 = Size0.getValue() + SrcValOffset0 - MinOffset; - int64_t Overlap1 = Size1.getValue() + SrcValOffset1 - MinOffset; + int64_t Overlap0 = + Size0.getValue().getKnownMinValue() + SrcValOffset0 - MinOffset; + int64_t Overlap1 = + Size1.getValue().getKnownMinValue() + SrcValOffset1 - MinOffset; + LocationSize Loc0 = + Size0.isScalable() ? Size0 : LocationSize::precise(Overlap0); + LocationSize Loc1 = + Size1.isScalable() ? Size1 : LocationSize::precise(Overlap1); if (AA->isNoAlias( - MemoryLocation(MUC0.MMO->getValue(), Overlap0, + MemoryLocation(MUC0.MMO->getValue(), Loc0, UseTBAA ? MUC0.MMO->getAAInfo() : AAMDNodes()), - MemoryLocation(MUC1.MMO->getValue(), Overlap1, + MemoryLocation(MUC1.MMO->getValue(), Loc1, UseTBAA ? MUC1.MMO->getAAInfo() : AAMDNodes()))) return false; } diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index e2c07e7cb997..0ab5142ab816 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -8406,9 +8406,7 @@ SDValue SelectionDAG::getMemIntrinsicNode( EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags, LocationSize Size, const AAMDNodes &AAInfo) { - if (Size.hasValue() && MemVT.isScalableVector()) - Size = LocationSize::beforeOrAfterPointer(); - else if (Size.hasValue() && !Size.getValue()) + if (Size.hasValue() && !Size.getValue()) Size = LocationSize::precise(MemVT.getStoreSize()); MachineFunction &MF = getMachineFunction(); @@ -8571,7 +8569,7 @@ SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, if (PtrInfo.V.isNull()) PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr, Offset); - LocationSize Size = MemoryLocation::getSizeOrUnknown(MemVT.getStoreSize()); + LocationSize Size = LocationSize::precise(MemVT.getStoreSize()); MachineFunction &MF = getMachineFunction(); MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, MMOFlags, Size, Alignment, AAInfo, Ranges); @@ -8692,8 +8690,7 @@ SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val, PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr); MachineFunction &MF = getMachineFunction(); - LocationSize Size = - MemoryLocation::getSizeOrUnknown(Val.getValueType().getStoreSize()); + LocationSize Size = LocationSize::precise(Val.getValueType().getStoreSize()); MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, MMOFlags, Size, Alignment, AAInfo); return getStore(Chain, dl, Val, Ptr, MMO); @@ -8746,8 +8743,8 @@ SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, MachineFunction &MF = getMachineFunction(); MachineMemOperand *MMO = MF.getMachineMemOperand( - PtrInfo, MMOFlags, MemoryLocation::getSizeOrUnknown(SVT.getStoreSize()), - Alignment, AAInfo); + PtrInfo, MMOFlags, LocationSize::precise(SVT.getStoreSize()), Alignment, + AAInfo); return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO); } @@ -8841,7 +8838,7 @@ SDValue SelectionDAG::getLoadVP( if (PtrInfo.V.isNull()) PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr, Offset); - LocationSize Size = MemoryLocation::getSizeOrUnknown(MemVT.getStoreSize()); + LocationSize Size = LocationSize::precise(MemVT.getStoreSize()); MachineFunction &MF = getMachineFunction(); MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, MMOFlags, Size, Alignment, AAInfo, Ranges); @@ -8994,8 +8991,8 @@ SDValue SelectionDAG::getTruncStoreVP(SDValue Chain, const SDLoc &dl, MachineFunction &MF = getMachineFunction(); MachineMemOperand *MMO = MF.getMachineMemOperand( - PtrInfo, MMOFlags, MemoryLocation::getSizeOrUnknown(SVT.getStoreSize()), - Alignment, AAInfo); + PtrInfo, MMOFlags, LocationSize::precise(SVT.getStoreSize()), Alignment, + AAInfo); return getTruncStoreVP(Chain, dl, Val, Ptr, Mask, EVL, SVT, MMO, IsCompressing); } @@ -11734,10 +11731,9 @@ MemSDNode::MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, // We check here that the size of the memory operand fits within the size of // the MMO. This is because the MMO might indicate only a possible address // range instead of specifying the affected memory addresses precisely. - // TODO: Make MachineMemOperands aware of scalable vectors. assert( (!MMO->getType().isValid() || - memvt.getStoreSize().getKnownMinValue() <= MMO->getSize().getValue()) && + TypeSize::isKnownLE(memvt.getStoreSize(), MMO->getSize().getValue())) && "Size mismatch!"); } diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp index 9670c3ac8430..f2ab88851b78 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp @@ -106,8 +106,6 @@ bool BaseIndexOffset::computeAliasing(const SDNode *Op0, int64_t PtrDiff; if (BasePtr0.equalBaseIndex(BasePtr1, DAG, PtrDiff)) { // If the size of memory access is unknown, do not use it to analysis. - // One example of unknown size memory access is to load/store scalable - // vector objects on the stack. // BasePtr1 is PtrDiff away from BasePtr0. They alias if none of the // following situations arise: if (PtrDiff >= 0 && NumBytes0.hasValue() && !NumBytes0.isScalable()) { diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 84df98b8a613..ae6bd7e938e8 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -4962,7 +4962,8 @@ void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) { unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace(); MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( MachinePointerInfo(AS), MachineMemOperand::MOLoad, - LocationSize::beforeOrAfterPointer(), Alignment, I.getAAMetadata(), Ranges); + LocationSize::beforeOrAfterPointer(), Alignment, I.getAAMetadata(), + Ranges); if (!UniformBase) { Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout())); diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp index 02943b8a4ab1..d0c5e6b99e9e 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp @@ -2687,10 +2687,7 @@ bool AArch64InstrInfo::getMemOperandsWithOffsetWidth( return false; // The maximum vscale is 16 under AArch64, return the maximal extent for the // vector. - Width = WidthN.isScalable() - ? WidthN.getKnownMinValue() * AArch64::SVEMaxBitsPerVector / - AArch64::SVEBitsPerBlock - : WidthN.getKnownMinValue(); + Width = LocationSize::precise(WidthN); BaseOps.push_back(BaseOp); return true; } diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 5a2fb0239e0a..5214595485ca 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -10364,9 +10364,15 @@ RISCVTargetLowering::lowerFixedLengthVectorLoadToRVV(SDValue Op, RISCVTargetLowering::computeVLMAXBounds(ContainerVT, Subtarget); if (MinVLMAX == MaxVLMAX && MinVLMAX == VT.getVectorNumElements() && getLMUL1VT(ContainerVT).bitsLE(ContainerVT)) { + MachineMemOperand *MMO = Load->getMemOperand(); + MachineFunction &MF = DAG.getMachineFunction(); + MMO = MF.getMachineMemOperand( + MMO, MMO->getPointerInfo(), + MMO->getMemoryType().isValid() + ? LLT::scalable_vector(1, MMO->getMemoryType().getSizeInBits()) + : MMO->getMemoryType()); SDValue NewLoad = - DAG.getLoad(ContainerVT, DL, Load->getChain(), Load->getBasePtr(), - Load->getMemOperand()); + DAG.getLoad(ContainerVT, DL, Load->getChain(), Load->getBasePtr(), MMO); SDValue Result = convertFromScalableVector(VT, NewLoad, DAG, Subtarget); return DAG.getMergeValues({Result, NewLoad.getValue(1)}, DL); } @@ -10424,9 +10430,17 @@ RISCVTargetLowering::lowerFixedLengthVectorStoreToRVV(SDValue Op, const auto [MinVLMAX, MaxVLMAX] = RISCVTargetLowering::computeVLMAXBounds(ContainerVT, Subtarget); if (MinVLMAX == MaxVLMAX && MinVLMAX == VT.getVectorNumElements() && - getLMUL1VT(ContainerVT).bitsLE(ContainerVT)) + getLMUL1VT(ContainerVT).bitsLE(ContainerVT)) { + MachineMemOperand *MMO = Store->getMemOperand(); + MachineFunction &MF = DAG.getMachineFunction(); + MMO = MF.getMachineMemOperand( + MMO, MMO->getPointerInfo(), + MMO->getMemoryType().isValid() + ? LLT::scalable_vector(1, MMO->getMemoryType().getSizeInBits()) + : MMO->getMemoryType()); return DAG.getStore(Store->getChain(), DL, NewValue, Store->getBasePtr(), - Store->getMemOperand()); + MMO); + } SDValue VL = getVLOp(VT.getVectorNumElements(), ContainerVT, DL, DAG, Subtarget); diff --git a/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll b/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll index 58299696e78f..d4d803a91cfa 100644 --- a/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll +++ b/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll @@ -3,7 +3,7 @@ define void @UphPNR(target("aarch64.svcount") %predcnt) { entry: ; CHECK: %0:ppr = COPY $p0 -; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store unknown-size into %ir.predcnt.addr, align 2) +; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store () into %ir.predcnt.addr) ; CHECK: %1:pnr_p8to15 = COPY %0 ; CHECK: INLINEASM &"ld1w {z0.s,z1.s,z2.s,z3.s}, $0/z, [x10]", 1 /* sideeffect attdialect */, {{[0-9]+}} /* reguse:PNR_p8to15 */, %1 ; CHECK: RET_ReallyLR @@ -17,7 +17,7 @@ entry: define void @UpaPNR(target("aarch64.svcount") %predcnt) { entry: ; CHECK: %0:ppr = COPY $p0 -; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store unknown-size into %ir.predcnt.addr, align 2) +; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store () into %ir.predcnt.addr) ; CHECK: %1:pnr = COPY %0 ; CHECK: INLINEASM &"ld1w {z0.s,z1.s,z2.s,z3.s}, $0/z, [x10]", 1 /* sideeffect attdialect */, {{[0-9]+}} /* reguse:PNR */, %1 ; CHECK: RET_ReallyLR @@ -31,7 +31,7 @@ entry: define void @UplPNR(target("aarch64.svcount") %predcnt) { entry: ; CHECK: %0:ppr = COPY $p0 -; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store unknown-size into %ir.predcnt.addr, align 2) +; CHECK: STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store () into %ir.predcnt.addr) ; CHECK: %1:pnr_3b = COPY %0 ; CHECK: INLINEASM &"fadd z0.h, $0/m, z0.h, #0.5", 1 /* sideeffect attdialect */, {{[0-9]+}} /* reguse:PNR_3b */, %1 ; CHECK: RET_ReallyLR diff --git a/llvm/test/CodeGen/AArch64/alloca-load-store-scalable-array.ll b/llvm/test/CodeGen/AArch64/alloca-load-store-scalable-array.ll index 9a4e01a29ecb..7244ac949ab8 100644 --- a/llvm/test/CodeGen/AArch64/alloca-load-store-scalable-array.ll +++ b/llvm/test/CodeGen/AArch64/alloca-load-store-scalable-array.ll @@ -14,12 +14,12 @@ define void @array_1D(ptr %addr) #0 { ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG ; CHECK-NEXT: .cfi_offset w29, -16 ; CHECK-NEXT: ptrue p0.d -; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, #2, mul vl] -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0, #1, mul vl] -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x0] -; CHECK-NEXT: st1d { z0.d }, p0, [sp, #2, mul vl] -; CHECK-NEXT: st1d { z1.d }, p0, [sp, #1, mul vl] -; CHECK-NEXT: st1d { z2.d }, p0, [sp] +; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0] +; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0, #2, mul vl] +; CHECK-NEXT: ld1d { z2.d }, p0/z, [x0, #1, mul vl] +; CHECK-NEXT: st1d { z0.d }, p0, [sp] +; CHECK-NEXT: st1d { z1.d }, p0, [sp, #2, mul vl] +; CHECK-NEXT: st1d { z2.d }, p0, [sp, #1, mul vl] ; CHECK-NEXT: addvl sp, sp, #3 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload ; CHECK-NEXT: ret @@ -81,18 +81,18 @@ define void @array_2D(ptr %addr) #0 { ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x30, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 48 * VG ; CHECK-NEXT: .cfi_offset w29, -16 ; CHECK-NEXT: ptrue p0.d -; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, #5, mul vl] -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0, #4, mul vl] -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x0] -; CHECK-NEXT: ld1d { z3.d }, p0/z, [x0, #3, mul vl] -; CHECK-NEXT: ld1d { z4.d }, p0/z, [x0, #1, mul vl] -; CHECK-NEXT: ld1d { z5.d }, p0/z, [x0, #2, mul vl] -; CHECK-NEXT: st1d { z0.d }, p0, [sp, #5, mul vl] -; CHECK-NEXT: st1d { z1.d }, p0, [sp, #4, mul vl] -; CHECK-NEXT: st1d { z3.d }, p0, [sp, #3, mul vl] -; CHECK-NEXT: st1d { z5.d }, p0, [sp, #2, mul vl] -; CHECK-NEXT: st1d { z4.d }, p0, [sp, #1, mul vl] -; CHECK-NEXT: st1d { z2.d }, p0, [sp] +; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0] +; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0, #5, mul vl] +; CHECK-NEXT: ld1d { z2.d }, p0/z, [x0, #1, mul vl] +; CHECK-NEXT: ld1d { z3.d }, p0/z, [x0, #4, mul vl] +; CHECK-NEXT: ld1d { z4.d }, p0/z, [x0, #2, mul vl] +; CHECK-NEXT: ld1d { z5.d }, p0/z, [x0, #3, mul vl] +; CHECK-NEXT: st1d { z0.d }, p0, [sp] +; CHECK-NEXT: st1d { z1.d }, p0, [sp, #5, mul vl] +; CHECK-NEXT: st1d { z3.d }, p0, [sp, #4, mul vl] +; CHECK-NEXT: st1d { z5.d }, p0, [sp, #3, mul vl] +; CHECK-NEXT: st1d { z4.d }, p0, [sp, #2, mul vl] +; CHECK-NEXT: st1d { z2.d }, p0, [sp, #1, mul vl] ; CHECK-NEXT: addvl sp, sp, #6 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/alloca-load-store-scalable-struct.ll b/llvm/test/CodeGen/AArch64/alloca-load-store-scalable-struct.ll index 7292d52aaf47..f03a6f018d34 100644 --- a/llvm/test/CodeGen/AArch64/alloca-load-store-scalable-struct.ll +++ b/llvm/test/CodeGen/AArch64/alloca-load-store-scalable-struct.ll @@ -13,12 +13,12 @@ define void @test(ptr %addr) #0 { ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG ; CHECK-NEXT: .cfi_offset w29, -16 ; CHECK-NEXT: ptrue p0.d -; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, #2, mul vl] -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0, #1, mul vl] -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x0] -; CHECK-NEXT: st1d { z0.d }, p0, [sp, #2, mul vl] -; CHECK-NEXT: st1d { z1.d }, p0, [sp, #1, mul vl] -; CHECK-NEXT: st1d { z2.d }, p0, [sp] +; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0] +; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0, #2, mul vl] +; CHECK-NEXT: ld1d { z2.d }, p0/z, [x0, #1, mul vl] +; CHECK-NEXT: st1d { z0.d }, p0, [sp] +; CHECK-NEXT: st1d { z1.d }, p0, [sp, #2, mul vl] +; CHECK-NEXT: st1d { z2.d }, p0, [sp, #1, mul vl] ; CHECK-NEXT: addvl sp, sp, #3 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/RISCV/rvv/alloca-load-store-scalable-array.ll b/llvm/test/CodeGen/RISCV/rvv/alloca-load-store-scalable-array.ll index 1d025a2f776f..1fe91c721f4d 100644 --- a/llvm/test/CodeGen/RISCV/rvv/alloca-load-store-scalable-array.ll +++ b/llvm/test/CodeGen/RISCV/rvv/alloca-load-store-scalable-array.ll @@ -18,15 +18,15 @@ define void @test(ptr %addr) { ; CHECK-NEXT: add a2, a0, a1 ; CHECK-NEXT: vl1re64.v v8, (a2) ; CHECK-NEXT: slli a2, a1, 1 -; CHECK-NEXT: add a3, a0, a2 -; CHECK-NEXT: vl1re64.v v9, (a3) +; CHECK-NEXT: vl1re64.v v9, (a0) +; CHECK-NEXT: add a0, a0, a2 ; CHECK-NEXT: vl1re64.v v10, (a0) ; CHECK-NEXT: addi a0, sp, 16 +; CHECK-NEXT: vs1r.v v9, (a0) ; CHECK-NEXT: add a2, a0, a2 -; CHECK-NEXT: vs1r.v v9, (a2) -; CHECK-NEXT: add a1, a0, a1 -; CHECK-NEXT: vs1r.v v8, (a1) -; CHECK-NEXT: vs1r.v v10, (a0) +; CHECK-NEXT: vs1r.v v10, (a2) +; CHECK-NEXT: add a0, a0, a1 +; CHECK-NEXT: vs1r.v v8, (a0) ; CHECK-NEXT: csrrs a0, vlenb, zero ; CHECK-NEXT: slli a0, a0, 2 ; CHECK-NEXT: add sp, sp, a0 diff --git a/llvm/test/CodeGen/RISCV/rvv/alloca-load-store-scalable-struct.ll b/llvm/test/CodeGen/RISCV/rvv/alloca-load-store-scalable-struct.ll index 64031f8a9359..a9a680d54d58 100644 --- a/llvm/test/CodeGen/RISCV/rvv/alloca-load-store-scalable-struct.ll +++ b/llvm/test/CodeGen/RISCV/rvv/alloca-load-store-scalable-struct.ll @@ -16,13 +16,13 @@ define @test(ptr %addr, i64 %vl) { ; CHECK-NEXT: sub sp, sp, a2 ; CHECK-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 2 * vlenb ; CHECK-NEXT: csrrs a2, vlenb, zero -; CHECK-NEXT: add a3, a0, a2 -; CHECK-NEXT: vl1re64.v v8, (a3) +; CHECK-NEXT: vl1re64.v v8, (a0) +; CHECK-NEXT: add a0, a0, a2 ; CHECK-NEXT: vl1re64.v v9, (a0) ; CHECK-NEXT: addi a0, sp, 16 +; CHECK-NEXT: vs1r.v v8, (a0) ; CHECK-NEXT: add a2, a0, a2 -; CHECK-NEXT: vs1r.v v8, (a2) -; CHECK-NEXT: vs1r.v v9, (a0) +; CHECK-NEXT: vs1r.v v9, (a2) ; CHECK-NEXT: vl1re64.v v8, (a2) ; CHECK-NEXT: vl1re64.v v9, (a0) ; CHECK-NEXT: vsetvli zero, a1, e64, m1, ta, ma diff --git a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll b/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll index 0544204cce79..52bd15742ef4 100644 --- a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll +++ b/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll @@ -16,8 +16,8 @@ define void @vpmerge_vpload_store( %passthru, ptr %p, ) into %ir.p) ; CHECK-NEXT: PseudoRET %splat = insertelement poison, i1 -1, i32 0 %mask = shufflevector %splat, poison, zeroinitializer @@ -37,8 +37,8 @@ define void @vpselect_vpload_store( %passthru, ptr %p, ) into %ir.p) ; CHECK-NEXT: PseudoRET %splat = insertelement poison, i1 -1, i32 0 %mask = shufflevector %splat, poison, zeroinitializer -- GitLab From b68e2eba0bc8dd70b88f4271831139ee9b6ed25c Mon Sep 17 00:00:00 2001 From: Nikolas Klauser Date: Sat, 23 Mar 2024 15:28:22 +0100 Subject: [PATCH 146/306] [libc++] Vectorize mismatch (#73255) ``` --------------------------------------------------- Benchmark old new --------------------------------------------------- bm_mismatch/1 0.835 ns 2.37 ns bm_mismatch/2 1.44 ns 2.60 ns bm_mismatch/3 2.06 ns 2.83 ns bm_mismatch/4 2.60 ns 3.29 ns bm_mismatch/5 3.15 ns 3.77 ns bm_mismatch/6 3.82 ns 4.17 ns bm_mismatch/7 4.29 ns 4.52 ns bm_mismatch/8 4.78 ns 4.86 ns bm_mismatch/16 9.06 ns 7.54 ns bm_mismatch/64 31.7 ns 19.1 ns bm_mismatch/512 249 ns 8.16 ns bm_mismatch/4096 1956 ns 44.2 ns bm_mismatch/32768 15498 ns 501 ns bm_mismatch/262144 123965 ns 4479 ns bm_mismatch/1048576 495668 ns 21306 ns bm_mismatch/1 0.710 ns 2.12 ns bm_mismatch/2 1.03 ns 2.66 ns bm_mismatch/3 1.29 ns 3.56 ns bm_mismatch/4 1.68 ns 4.29 ns bm_mismatch/5 1.96 ns 5.18 ns bm_mismatch/6 2.59 ns 5.91 ns bm_mismatch/7 2.86 ns 6.63 ns bm_mismatch/8 3.19 ns 7.33 ns bm_mismatch/16 5.48 ns 13.0 ns bm_mismatch/64 16.6 ns 4.06 ns bm_mismatch/512 130 ns 13.8 ns bm_mismatch/4096 985 ns 93.8 ns bm_mismatch/32768 7846 ns 1002 ns bm_mismatch/262144 63217 ns 10637 ns bm_mismatch/1048576 251782 ns 42471 ns bm_mismatch/1 0.716 ns 1.91 ns bm_mismatch/2 1.21 ns 2.49 ns bm_mismatch/3 1.38 ns 3.46 ns bm_mismatch/4 1.71 ns 4.04 ns bm_mismatch/5 2.00 ns 4.98 ns bm_mismatch/6 2.43 ns 5.67 ns bm_mismatch/7 3.05 ns 6.38 ns bm_mismatch/8 3.22 ns 7.09 ns bm_mismatch/16 5.18 ns 12.8 ns bm_mismatch/64 16.6 ns 5.28 ns bm_mismatch/512 129 ns 25.2 ns bm_mismatch/4096 1009 ns 201 ns bm_mismatch/32768 7776 ns 2144 ns bm_mismatch/262144 62371 ns 20551 ns bm_mismatch/1048576 254750 ns 90097 ns ``` --- libcxx/benchmarks/CMakeLists.txt | 1 + .../benchmarks/algorithms/mismatch.bench.cpp | 31 +++ libcxx/docs/ReleaseNotes/19.rst | 2 + libcxx/include/CMakeLists.txt | 1 + libcxx/include/__algorithm/mismatch.h | 82 ++++++- libcxx/include/__algorithm/simd_utils.h | 123 ++++++++++ libcxx/include/__bit/bit_cast.h | 9 + libcxx/include/__bit/countr.h | 13 +- libcxx/include/libcxx.imp | 1 + libcxx/include/module.modulemap | 6 +- .../mismatch/mismatch.pass.cpp | 214 +++++++++++++----- .../mismatch/mismatch_pred.pass.cpp | 119 ---------- 12 files changed, 413 insertions(+), 189 deletions(-) create mode 100644 libcxx/benchmarks/algorithms/mismatch.bench.cpp create mode 100644 libcxx/include/__algorithm/simd_utils.h delete mode 100644 libcxx/test/std/algorithms/alg.nonmodifying/mismatch/mismatch_pred.pass.cpp diff --git a/libcxx/benchmarks/CMakeLists.txt b/libcxx/benchmarks/CMakeLists.txt index 3dec6faea13a..387e013afeb6 100644 --- a/libcxx/benchmarks/CMakeLists.txt +++ b/libcxx/benchmarks/CMakeLists.txt @@ -183,6 +183,7 @@ set(BENCHMARK_TESTS algorithms/make_heap_then_sort_heap.bench.cpp algorithms/min.bench.cpp algorithms/min_max_element.bench.cpp + algorithms/mismatch.bench.cpp algorithms/pop_heap.bench.cpp algorithms/pstl.stable_sort.bench.cpp algorithms/push_heap.bench.cpp diff --git a/libcxx/benchmarks/algorithms/mismatch.bench.cpp b/libcxx/benchmarks/algorithms/mismatch.bench.cpp new file mode 100644 index 000000000000..9274932a764c --- /dev/null +++ b/libcxx/benchmarks/algorithms/mismatch.bench.cpp @@ -0,0 +1,31 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include +#include +#include + +// TODO: Look into benchmarking aligned and unaligned memory explicitly +// (currently things happen to be aligned because they are malloced that way) +template +static void bm_mismatch(benchmark::State& state) { + std::vector vec1(state.range(), '1'); + std::vector vec2(state.range(), '1'); + std::mt19937_64 rng(std::random_device{}()); + + vec1.back() = '2'; + for (auto _ : state) { + benchmark::DoNotOptimize(vec1); + benchmark::DoNotOptimize(std::mismatch(vec1.begin(), vec1.end(), vec2.begin())); + } +} +BENCHMARK(bm_mismatch)->DenseRange(1, 8)->Range(16, 1 << 20); +BENCHMARK(bm_mismatch)->DenseRange(1, 8)->Range(16, 1 << 20); +BENCHMARK(bm_mismatch)->DenseRange(1, 8)->Range(16, 1 << 20); + +BENCHMARK_MAIN(); diff --git a/libcxx/docs/ReleaseNotes/19.rst b/libcxx/docs/ReleaseNotes/19.rst index cac42f9c3c3f..dd39c1bbbc78 100644 --- a/libcxx/docs/ReleaseNotes/19.rst +++ b/libcxx/docs/ReleaseNotes/19.rst @@ -51,6 +51,8 @@ Improvements and New Features - The performance of growing ``std::vector`` has been improved for trivially relocatable types. - The performance of ``ranges::fill`` and ``ranges::fill_n`` has been improved for ``vector::iterator``\s, resulting in a performance increase of up to 1400x. +- The ``std::mismatch`` algorithm has been optimized for integral types, which can lead up to 40x performance + improvements. Deprecations and Removals ------------------------- diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt index 6ed8d21d98a1..982b85e4e2d6 100644 --- a/libcxx/include/CMakeLists.txt +++ b/libcxx/include/CMakeLists.txt @@ -217,6 +217,7 @@ set(files __algorithm/shift_right.h __algorithm/shuffle.h __algorithm/sift_down.h + __algorithm/simd_utils.h __algorithm/sort.h __algorithm/sort_heap.h __algorithm/stable_partition.h diff --git a/libcxx/include/__algorithm/mismatch.h b/libcxx/include/__algorithm/mismatch.h index d345b6048a7e..4eb693a1f2e9 100644 --- a/libcxx/include/__algorithm/mismatch.h +++ b/libcxx/include/__algorithm/mismatch.h @@ -11,23 +11,93 @@ #define _LIBCPP___ALGORITHM_MISMATCH_H #include <__algorithm/comp.h> +#include <__algorithm/simd_utils.h> +#include <__algorithm/unwrap_iter.h> #include <__config> -#include <__iterator/iterator_traits.h> +#include <__functional/identity.h> +#include <__type_traits/invoke.h> +#include <__type_traits/is_constant_evaluated.h> +#include <__type_traits/is_equality_comparable.h> +#include <__type_traits/operation_traits.h> +#include <__utility/move.h> #include <__utility/pair.h> +#include <__utility/unreachable.h> #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) # pragma GCC system_header #endif +_LIBCPP_PUSH_MACROS +#include <__undef_macros> + _LIBCPP_BEGIN_NAMESPACE_STD +template +_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_Iter1, _Iter2> +__mismatch_loop(_Iter1 __first1, _Sent1 __last1, _Iter2 __first2, _Pred& __pred, _Proj1& __proj1, _Proj2& __proj2) { + while (__first1 != __last1) { + if (!std::__invoke(__pred, std::__invoke(__proj1, *__first1), std::__invoke(__proj2, *__first2))) + break; + ++__first1; + ++__first2; + } + return std::make_pair(std::move(__first1), std::move(__first2)); +} + +template +_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_Iter1, _Iter2> +__mismatch(_Iter1 __first1, _Sent1 __last1, _Iter2 __first2, _Pred& __pred, _Proj1& __proj1, _Proj2& __proj2) { + return std::__mismatch_loop(__first1, __last1, __first2, __pred, __proj1, __proj2); +} + +#if _LIBCPP_VECTORIZE_ALGORITHMS + +template ::value && __desugars_to<__equal_tag, _Pred, _Tp, _Tp>::value && + __is_identity<_Proj1>::value && __is_identity<_Proj2>::value, + int> = 0> +_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_Tp*, _Tp*> +__mismatch(_Tp* __first1, _Tp* __last1, _Tp* __first2, _Pred& __pred, _Proj1& __proj1, _Proj2& __proj2) { + constexpr size_t __unroll_count = 4; + constexpr size_t __vec_size = __native_vector_size<_Tp>; + using __vec = __simd_vector<_Tp, __vec_size>; + if (!__libcpp_is_constant_evaluated()) { + while (static_cast(__last1 - __first1) >= __unroll_count * __vec_size) [[__unlikely__]] { + __vec __lhs[__unroll_count]; + __vec __rhs[__unroll_count]; + + for (size_t __i = 0; __i != __unroll_count; ++__i) { + __lhs[__i] = std::__load_vector<__vec>(__first1 + __i * __vec_size); + __rhs[__i] = std::__load_vector<__vec>(__first2 + __i * __vec_size); + } + + for (size_t __i = 0; __i != __unroll_count; ++__i) { + if (auto __cmp_res = __lhs[__i] == __rhs[__i]; !std::__all_of(__cmp_res)) { + auto __offset = __i * __vec_size + std::__find_first_not_set(__cmp_res); + return {__first1 + __offset, __first2 + __offset}; + } + } + + __first1 += __unroll_count * __vec_size; + __first2 += __unroll_count * __vec_size; + } + } + // TODO: Consider vectorizing the tail + return std::__mismatch_loop(__first1, __last1, __first2, __pred, __proj1, __proj2); +} + +#endif // _LIBCPP_VECTORIZE_ALGORITHMS + template _LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_InputIterator1, _InputIterator2> mismatch(_InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first2, _BinaryPredicate __pred) { - for (; __first1 != __last1; ++__first1, (void)++__first2) - if (!__pred(*__first1, *__first2)) - break; - return pair<_InputIterator1, _InputIterator2>(__first1, __first2); + __identity __proj; + auto __res = std::__mismatch( + std::__unwrap_iter(__first1), std::__unwrap_iter(__last1), std::__unwrap_iter(__first2), __pred, __proj, __proj); + return std::make_pair(std::__rewrap_iter(__first1, __res.first), std::__rewrap_iter(__first2, __res.second)); } template @@ -59,4 +129,6 @@ mismatch(_InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __fi _LIBCPP_END_NAMESPACE_STD +_LIBCPP_POP_MACROS + #endif // _LIBCPP___ALGORITHM_MISMATCH_H diff --git a/libcxx/include/__algorithm/simd_utils.h b/libcxx/include/__algorithm/simd_utils.h new file mode 100644 index 000000000000..1aedb3db010f --- /dev/null +++ b/libcxx/include/__algorithm/simd_utils.h @@ -0,0 +1,123 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ALGORITHM_SIMD_UTILS_H +#define _LIBCPP___ALGORITHM_SIMD_UTILS_H + +#include <__bit/bit_cast.h> +#include <__bit/countr.h> +#include <__config> +#include <__type_traits/is_arithmetic.h> +#include <__type_traits/is_same.h> +#include <__utility/integer_sequence.h> +#include +#include + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +// TODO: Find out how altivec changes things and allow vectorizations there too. +#if _LIBCPP_STD_VER >= 14 && defined(_LIBCPP_CLANG_VER) && _LIBCPP_CLANG_VER >= 1700 && !defined(__ALTIVEC__) +# define _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS 1 +#else +# define _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS 0 +#endif + +#if _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS && !defined(__OPTIMIZE_SIZE__) +# define _LIBCPP_VECTORIZE_ALGORITHMS 1 +#else +# define _LIBCPP_VECTORIZE_ALGORITHMS 0 +#endif + +#if _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS + +_LIBCPP_BEGIN_NAMESPACE_STD + +// This isn't specialized for 64 byte vectors on purpose. They have the potential to significantly reduce performance +// in mixed simd/non-simd workloads and don't provide any performance improvement for currently vectorized algorithms +// as far as benchmarks are concerned. +# if defined(__AVX__) +template +inline constexpr size_t __native_vector_size = 32 / sizeof(_Tp); +# elif defined(__SSE__) || defined(__ARM_NEON__) +template +inline constexpr size_t __native_vector_size = 16 / sizeof(_Tp); +# elif defined(__MMX__) +template +inline constexpr size_t __native_vector_size = 8 / sizeof(_Tp); +# else +template +inline constexpr size_t __native_vector_size = 1; +# endif + +template +using __simd_vector __attribute__((__ext_vector_type__(_Np))) = _ArithmeticT; + +template +inline constexpr size_t __simd_vector_size_v = []() -> size_t { + static_assert(_False, "Not a vector!"); +}(); + +template +inline constexpr size_t __simd_vector_size_v<__simd_vector<_Tp, _Np>> = _Np; + +template +_LIBCPP_HIDE_FROM_ABI _Tp __simd_vector_underlying_type_impl(__simd_vector<_Tp, _Np>) { + return _Tp{}; +} + +template +using __simd_vector_underlying_type_t = decltype(std::__simd_vector_underlying_type_impl(_VecT{})); + +// This isn't inlined without always_inline when loading chars. +template +_LIBCPP_NODISCARD _LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI _VecT __load_vector(const _Tp* __ptr) noexcept { + return [=](index_sequence<_Indices...>) _LIBCPP_ALWAYS_INLINE noexcept { + return _VecT{__ptr[_Indices]...}; + }(make_index_sequence<__simd_vector_size_v<_VecT>>{}); +} + +template +_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI bool __all_of(__simd_vector<_Tp, _Np> __vec) noexcept { + return __builtin_reduce_and(__builtin_convertvector(__vec, __simd_vector)); +} + +template +_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI size_t __find_first_set(__simd_vector<_Tp, _Np> __vec) noexcept { + using __mask_vec = __simd_vector; + + // This has MSan disabled du to https://github.com/llvm/llvm-project/issues/85876 + auto __impl = [&](_MaskT) _LIBCPP_NO_SANITIZE("memory") noexcept { + return std::__countr_zero(__builtin_bit_cast(_MaskT, __builtin_convertvector(__vec, __mask_vec))); + }; + + if constexpr (sizeof(__mask_vec) == sizeof(uint8_t)) { + return __impl(uint8_t{}); + } else if constexpr (sizeof(__mask_vec) == sizeof(uint16_t)) { + return __impl(uint16_t{}); + } else if constexpr (sizeof(__mask_vec) == sizeof(uint32_t)) { + return __impl(uint32_t{}); + } else if constexpr (sizeof(__mask_vec) == sizeof(uint64_t)) { + return __impl(uint64_t{}); + } else { + static_assert(sizeof(__mask_vec) == 0, "unexpected required size for mask integer type"); + return 0; + } +} + +template +_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI size_t __find_first_not_set(__simd_vector<_Tp, _Np> __vec) noexcept { + return std::__find_first_set(~__vec); +} + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS + +#endif // _LIBCPP___ALGORITHM_SIMD_UTILS_H diff --git a/libcxx/include/__bit/bit_cast.h b/libcxx/include/__bit/bit_cast.h index f20b39ae748b..6298810f3733 100644 --- a/libcxx/include/__bit/bit_cast.h +++ b/libcxx/include/__bit/bit_cast.h @@ -19,6 +19,15 @@ _LIBCPP_BEGIN_NAMESPACE_STD +#ifndef _LIBCPP_CXX03_LANG + +template +_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI constexpr _ToType __bit_cast(const _FromType& __from) noexcept { + return __builtin_bit_cast(_ToType, __from); +} + +#endif // _LIBCPP_CXX03_LANG + #if _LIBCPP_STD_VER >= 20 template diff --git a/libcxx/include/__bit/countr.h b/libcxx/include/__bit/countr.h index 0cc679f87a99..b6b3ac52ca4e 100644 --- a/libcxx/include/__bit/countr.h +++ b/libcxx/include/__bit/countr.h @@ -35,10 +35,8 @@ _LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR int __libcpp_ct return __builtin_ctzll(__x); } -#if _LIBCPP_STD_VER >= 20 - -template <__libcpp_unsigned_integer _Tp> -_LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr int countr_zero(_Tp __t) noexcept { +template +_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 int __countr_zero(_Tp __t) _NOEXCEPT { if (__t == 0) return numeric_limits<_Tp>::digits; @@ -59,6 +57,13 @@ _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr int countr_zero(_Tp __t) n } } +#if _LIBCPP_STD_VER >= 20 + +template <__libcpp_unsigned_integer _Tp> +_LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr int countr_zero(_Tp __t) noexcept { + return std::__countr_zero(__t); +} + template <__libcpp_unsigned_integer _Tp> _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr int countr_one(_Tp __t) noexcept { return __t != numeric_limits<_Tp>::max() ? std::countr_zero(static_cast<_Tp>(~__t)) : numeric_limits<_Tp>::digits; diff --git a/libcxx/include/libcxx.imp b/libcxx/include/libcxx.imp index 77b7befd44f5..56ea58262828 100644 --- a/libcxx/include/libcxx.imp +++ b/libcxx/include/libcxx.imp @@ -217,6 +217,7 @@ { include: [ "<__algorithm/shift_right.h>", "private", "", "public" ] }, { include: [ "<__algorithm/shuffle.h>", "private", "", "public" ] }, { include: [ "<__algorithm/sift_down.h>", "private", "", "public" ] }, + { include: [ "<__algorithm/simd_utils.h>", "private", "", "public" ] }, { include: [ "<__algorithm/sort.h>", "private", "", "public" ] }, { include: [ "<__algorithm/sort_heap.h>", "private", "", "public" ] }, { include: [ "<__algorithm/stable_partition.h>", "private", "", "public" ] }, diff --git a/libcxx/include/module.modulemap b/libcxx/include/module.modulemap index f36a47cef009..03d18775631e 100644 --- a/libcxx/include/module.modulemap +++ b/libcxx/include/module.modulemap @@ -697,7 +697,10 @@ module std_private_algorithm_minmax [system export * } module std_private_algorithm_minmax_element [system] { header "__algorithm/minmax_element.h" } -module std_private_algorithm_mismatch [system] { header "__algorithm/mismatch.h" } +module std_private_algorithm_mismatch [system] { + header "__algorithm/mismatch.h" + export std_private_algorithm_simd_utils +} module std_private_algorithm_move [system] { header "__algorithm/move.h" } module std_private_algorithm_move_backward [system] { header "__algorithm/move_backward.h" } module std_private_algorithm_next_permutation [system] { header "__algorithm/next_permutation.h" } @@ -1048,6 +1051,7 @@ module std_private_algorithm_sort [system header "__algorithm/sort.h" export std_private_debug_utils_strict_weak_ordering_check } +module std_private_algorithm_simd_utils [system] { header "__algorithm/simd_utils.h" } module std_private_algorithm_sort_heap [system] { header "__algorithm/sort_heap.h" } module std_private_algorithm_stable_partition [system] { header "__algorithm/stable_partition.h" } module std_private_algorithm_stable_sort [system] { header "__algorithm/stable_sort.h" } diff --git a/libcxx/test/std/algorithms/alg.nonmodifying/mismatch/mismatch.pass.cpp b/libcxx/test/std/algorithms/alg.nonmodifying/mismatch/mismatch.pass.cpp index cc588c095ccf..e7f3994d977d 100644 --- a/libcxx/test/std/algorithms/alg.nonmodifying/mismatch/mismatch.pass.cpp +++ b/libcxx/test/std/algorithms/alg.nonmodifying/mismatch/mismatch.pass.cpp @@ -16,79 +16,173 @@ // template // constexpr pair // constexpr after c++17 // mismatch(Iter1 first1, Iter1 last1, Iter2 first2, Iter2 last2); // C++14 +// +// template Pred> +// requires CopyConstructible +// constexpr pair // constexpr after c++17 +// mismatch(Iter1 first1, Iter1 last1, Iter2 first2, Pred pred); +// +// template +// constexpr pair // constexpr after c++17 +// mismatch(Iter1 first1, Iter1 last1, Iter2 first2, Iter2 last2, Pred pred); // C++14 + +// ADDITIONAL_COMPILE_FLAGS(has-fconstexpr-steps): -fconstexpr-steps=50000000 +// ADDITIONAL_COMPILE_FLAGS(has-fconstexpr-ops-limit): -fconstexpr-ops-limit=100000000 #include +#include #include +#include #include "test_macros.h" #include "test_iterators.h" - -#if TEST_STD_VER > 17 -TEST_CONSTEXPR bool test_constexpr() { - int ia[] = {1, 3, 6, 7}; - int ib[] = {1, 3}; - int ic[] = {1, 3, 5, 7}; - typedef cpp17_input_iterator II; - typedef bidirectional_iterator BI; - - auto p1 = std::mismatch(std::begin(ia), std::end(ia), std::begin(ic)); - if (p1.first != ia+2 || p1.second != ic+2) - return false; - - auto p2 = std::mismatch(std::begin(ia), std::end(ia), std::begin(ic), std::end(ic)); - if (p2.first != ia+2 || p2.second != ic+2) - return false; - - auto p3 = std::mismatch(std::begin(ib), std::end(ib), std::begin(ic)); - if (p3.first != ib+2 || p3.second != ic+2) - return false; - - auto p4 = std::mismatch(std::begin(ib), std::end(ib), std::begin(ic), std::end(ic)); - if (p4.first != ib+2 || p4.second != ic+2) - return false; - - auto p5 = std::mismatch(II(std::begin(ib)), II(std::end(ib)), II(std::begin(ic))); - if (p5.first != II(ib+2) || p5.second != II(ic+2)) - return false; - auto p6 = std::mismatch(BI(std::begin(ib)), BI(std::end(ib)), BI(std::begin(ic)), BI(std::end(ic))); - if (p6.first != BI(ib+2) || p6.second != BI(ic+2)) - return false; - - return true; - } +#include "type_algorithms.h" + +template +TEST_CONSTEXPR_CXX20 void check(Container1 lhs, Container2 rhs, size_t offset) { + if (lhs.size() == rhs.size()) { + assert(std::mismatch(Iter(lhs.data()), Iter(lhs.data() + lhs.size()), Iter(rhs.data())) == + std::make_pair(Iter(lhs.data() + offset), Iter(rhs.data() + offset))); + + assert(std::mismatch(Iter(lhs.data()), + Iter(lhs.data() + lhs.size()), + Iter(rhs.data()), + std::equal_to()) == + std::make_pair(Iter(lhs.data() + offset), Iter(rhs.data() + offset))); + } + +#if TEST_STD_VER >= 14 + assert( + std::mismatch(Iter(lhs.data()), Iter(lhs.data() + lhs.size()), Iter(rhs.data()), Iter(rhs.data() + rhs.size())) == + std::make_pair(Iter(lhs.data() + offset), Iter(rhs.data() + offset))); + + assert(std::mismatch(Iter(lhs.data()), + Iter(lhs.data() + lhs.size()), + Iter(rhs.data()), + Iter(rhs.data() + rhs.size()), + std::equal_to()) == + std::make_pair(Iter(lhs.data() + offset), Iter(rhs.data() + offset))); #endif +} -int main(int, char**) -{ - int ia[] = {0, 1, 2, 2, 0, 1, 2, 3}; - const unsigned sa = sizeof(ia)/sizeof(ia[0]); - int ib[] = {0, 1, 2, 3, 0, 1, 2, 3}; - const unsigned sb = sizeof(ib)/sizeof(ib[0]); ((void)sb); // unused in C++11 - - typedef cpp17_input_iterator II; - typedef random_access_iterator RAI; - - assert(std::mismatch(II(ia), II(ia + sa), II(ib)) - == (std::pair(II(ia+3), II(ib+3)))); - - assert(std::mismatch(RAI(ia), RAI(ia + sa), RAI(ib)) - == (std::pair(RAI(ia+3), RAI(ib+3)))); - -#if TEST_STD_VER > 11 // We have the four iteration version - assert(std::mismatch(II(ia), II(ia + sa), II(ib), II(ib+sb)) - == (std::pair(II(ia+3), II(ib+3)))); +struct NonTrivial { + int i_; + + TEST_CONSTEXPR_CXX20 NonTrivial(int i) : i_(i) {} + TEST_CONSTEXPR_CXX20 NonTrivial(NonTrivial&& other) : i_(other.i_) { other.i_ = 0; } + + TEST_CONSTEXPR_CXX20 friend bool operator==(const NonTrivial& lhs, const NonTrivial& rhs) { return lhs.i_ == rhs.i_; } +}; + +struct ModTwoComp { + TEST_CONSTEXPR_CXX20 bool operator()(int lhs, int rhs) { return lhs % 2 == rhs % 2; } +}; + +template +TEST_CONSTEXPR_CXX20 bool test() { + { // empty ranges + std::array lhs = {}; + std::array rhs = {}; + check(lhs, rhs, 0); + } + + { // same range without mismatch + std::array lhs = {0, 1, 2, 3, 0, 1, 2, 3}; + std::array rhs = {0, 1, 2, 3, 0, 1, 2, 3}; + check(lhs, rhs, 8); + } + + { // same range with mismatch + std::array lhs = {0, 1, 2, 2, 0, 1, 2, 3}; + std::array rhs = {0, 1, 2, 3, 0, 1, 2, 3}; + check(lhs, rhs, 3); + } + + { // second range is smaller + std::array lhs = {0, 1, 2, 2, 0, 1, 2, 3}; + std::array rhs = {0, 1}; + check(lhs, rhs, 2); + } + + { // first range is smaller + std::array lhs = {0, 1}; + std::array rhs = {0, 1, 2, 2, 0, 1, 2, 3}; + check(lhs, rhs, 2); + } + + { // use a custom comparator + std::array lhs = {0, 2, 3, 4}; + std::array rhs = {0, 0, 4, 4}; + assert(std::mismatch(lhs.data(), lhs.data() + lhs.size(), rhs.data(), ModTwoComp()) == + std::make_pair(lhs.data() + 2, rhs.data() + 2)); +#if TEST_STD_VER >= 14 + assert(std::mismatch(lhs.data(), lhs.data() + lhs.size(), rhs.data(), rhs.data() + rhs.size(), ModTwoComp()) == + std::make_pair(lhs.data() + 2, rhs.data() + 2)); +#endif + } - assert(std::mismatch(RAI(ia), RAI(ia + sa), RAI(ib), RAI(ib+sb)) - == (std::pair(RAI(ia+3), RAI(ib+3)))); + return true; +} +struct Test { + template + TEST_CONSTEXPR_CXX20 void operator()() { + test(); + } +}; + +TEST_CONSTEXPR_CXX20 bool test() { + types::for_each(types::cpp17_input_iterator_list(), Test()); + + { // use a non-integer type to also test the general case - all elements match + std::array lhs = {1, 2, 3, 4, 5, 6, 7, 8}; + std::array rhs = {1, 2, 3, 4, 5, 6, 7, 8}; + check(std::move(lhs), std::move(rhs), 8); + } + + { // use a non-integer type to also test the general case - not all elements match + std::array lhs = {1, 2, 3, 4, 7, 6, 7, 8}; + std::array rhs = {1, 2, 3, 4, 5, 6, 7, 8}; + check(std::move(lhs), std::move(rhs), 4); + } + + return true; +} - assert(std::mismatch(II(ia), II(ia + sa), II(ib), II(ib+2)) - == (std::pair(II(ia+2), II(ib+2)))); +int main(int, char**) { + test(); +#if TEST_STD_VER >= 20 + static_assert(test()); #endif -#if TEST_STD_VER > 17 - static_assert(test_constexpr()); -#endif + { // check with a lot of elements to test the vectorization optimization + { + std::vector lhs(256); + std::vector rhs(256); + for (size_t i = 0; i != lhs.size(); ++i) { + lhs[i] = 1; + check(lhs, rhs, i); + lhs[i] = 0; + rhs[i] = 1; + check(lhs, rhs, i); + rhs[i] = 0; + } + } + + { + std::vector lhs(256); + std::vector rhs(256); + for (size_t i = 0; i != lhs.size(); ++i) { + lhs[i] = 1; + check(lhs, rhs, i); + lhs[i] = 0; + rhs[i] = 1; + check(lhs, rhs, i); + rhs[i] = 0; + } + } + } return 0; } diff --git a/libcxx/test/std/algorithms/alg.nonmodifying/mismatch/mismatch_pred.pass.cpp b/libcxx/test/std/algorithms/alg.nonmodifying/mismatch/mismatch_pred.pass.cpp deleted file mode 100644 index bda4ec7ba5ed..000000000000 --- a/libcxx/test/std/algorithms/alg.nonmodifying/mismatch/mismatch_pred.pass.cpp +++ /dev/null @@ -1,119 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -// - -// template Pred> -// requires CopyConstructible -// constexpr pair // constexpr after c++17 -// mismatch(Iter1 first1, Iter1 last1, Iter2 first2, Pred pred); -// -// template -// constexpr pair // constexpr after c++17 -// mismatch(Iter1 first1, Iter1 last1, Iter2 first2, Iter2 last2, Pred pred); // C++14 - -#include -#include -#include - -#include "test_macros.h" -#include "test_iterators.h" -#include "counting_predicates.h" - -#if TEST_STD_VER > 17 -TEST_CONSTEXPR bool eq(int a, int b) { return a == b; } - -TEST_CONSTEXPR bool test_constexpr() { - int ia[] = {1, 3, 6, 7}; - int ib[] = {1, 3}; - int ic[] = {1, 3, 5, 7}; - typedef cpp17_input_iterator II; - typedef bidirectional_iterator BI; - - auto p1 = std::mismatch(std::begin(ia), std::end(ia), std::begin(ic), eq); - if (p1.first != ia+2 || p1.second != ic+2) - return false; - - auto p2 = std::mismatch(std::begin(ia), std::end(ia), std::begin(ic), std::end(ic), eq); - if (p2.first != ia+2 || p2.second != ic+2) - return false; - - auto p3 = std::mismatch(std::begin(ib), std::end(ib), std::begin(ic), eq); - if (p3.first != ib+2 || p3.second != ic+2) - return false; - - auto p4 = std::mismatch(std::begin(ib), std::end(ib), std::begin(ic), std::end(ic), eq); - if (p4.first != ib+2 || p4.second != ic+2) - return false; - - auto p5 = std::mismatch(II(std::begin(ib)), II(std::end(ib)), II(std::begin(ic)), eq); - if (p5.first != II(ib+2) || p5.second != II(ic+2)) - return false; - auto p6 = std::mismatch(BI(std::begin(ib)), BI(std::end(ib)), BI(std::begin(ic)), BI(std::end(ic)), eq); - if (p6.first != BI(ib+2) || p6.second != BI(ic+2)) - return false; - - return true; - } -#endif - - -#if TEST_STD_VER > 11 -#define HAS_FOUR_ITERATOR_VERSION -#endif - -int main(int, char**) -{ - int ia[] = {0, 1, 2, 2, 0, 1, 2, 3}; - const unsigned sa = sizeof(ia)/sizeof(ia[0]); - int ib[] = {0, 1, 2, 3, 0, 1, 2, 3}; - const unsigned sb = sizeof(ib)/sizeof(ib[0]); ((void)sb); // unused in C++11 - - typedef cpp17_input_iterator II; - typedef random_access_iterator RAI; - typedef std::equal_to EQ; - - assert(std::mismatch(II(ia), II(ia + sa), II(ib), EQ()) - == (std::pair(II(ia+3), II(ib+3)))); - assert(std::mismatch(RAI(ia), RAI(ia + sa), RAI(ib), EQ()) - == (std::pair(RAI(ia+3), RAI(ib+3)))); - - binary_counting_predicate bcp((EQ())); - assert(std::mismatch(RAI(ia), RAI(ia + sa), RAI(ib), std::ref(bcp)) - == (std::pair(RAI(ia+3), RAI(ib+3)))); - assert(bcp.count() > 0 && bcp.count() < sa); - bcp.reset(); - -#if TEST_STD_VER >= 14 - assert(std::mismatch(II(ia), II(ia + sa), II(ib), II(ib + sb), EQ()) - == (std::pair(II(ia+3), II(ib+3)))); - assert(std::mismatch(RAI(ia), RAI(ia + sa), RAI(ib), RAI(ib + sb), EQ()) - == (std::pair(RAI(ia+3), RAI(ib+3)))); - - assert(std::mismatch(II(ia), II(ia + sa), II(ib), II(ib + sb), std::ref(bcp)) - == (std::pair(II(ia+3), II(ib+3)))); - assert(bcp.count() > 0 && bcp.count() < std::min(sa, sb)); -#endif - - assert(std::mismatch(ia, ia + sa, ib, EQ()) == - (std::pair(ia+3,ib+3))); - -#if TEST_STD_VER >= 14 - assert(std::mismatch(ia, ia + sa, ib, ib + sb, EQ()) == - (std::pair(ia+3,ib+3))); - assert(std::mismatch(ia, ia + sa, ib, ib + 2, EQ()) == - (std::pair(ia+2,ib+2))); -#endif - -#if TEST_STD_VER > 17 - static_assert(test_constexpr()); -#endif - - return 0; -} -- GitLab From 3f5e649ff64a92a732027ef76e33a1c8b1722d84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Sat, 23 Mar 2024 16:46:32 +0100 Subject: [PATCH 147/306] [Libomptarget] Fix linking to LLVM dylib (#86397) Use `LINK_COMPONENTS` parameter of `add_llvm_library` rather than passing LLVM components directly to `target_link_libraries`, in order to ensure that LLVM dylib is linked correctly when used. Otherwise, CMake insists on linking to static libraries that aren't present on distributions doing pure dylib installs, such as Gentoo. This fixes a regression introduced in dcbddc25250158469c5635ad2ae4095faef53dfd. --- .../plugins-nextgen/CMakeLists.txt | 55 +++++++++---------- 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/openmp/libomptarget/plugins-nextgen/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/CMakeLists.txt index c19fd80592d6..dbd82ac94517 100644 --- a/openmp/libomptarget/plugins-nextgen/CMakeLists.txt +++ b/openmp/libomptarget/plugins-nextgen/CMakeLists.txt @@ -14,41 +14,40 @@ set(common_dir ${CMAKE_CURRENT_SOURCE_DIR}/common) add_subdirectory(common) function(add_target_library target_name lib_name) - llvm_map_components_to_libnames(llvm_libs - ${LLVM_TARGETS_TO_BUILD} - AggressiveInstCombine - Analysis - BinaryFormat - BitReader - BitWriter - CodeGen - Core - Extensions - InstCombine - Instrumentation - IPO - IRReader - Linker - MC - Object - Passes - Remarks - ScalarOpts - Support - Target - TargetParser - TransformUtils - Vectorize - ) - add_llvm_library(${target_name} SHARED + LINK_COMPONENTS + ${LLVM_TARGETS_TO_BUILD} + AggressiveInstCombine + Analysis + BinaryFormat + BitReader + BitWriter + CodeGen + Core + Extensions + InstCombine + Instrumentation + IPO + IRReader + Linker + MC + Object + Passes + Remarks + ScalarOpts + Support + Target + TargetParser + TransformUtils + Vectorize + NO_INSTALL_RPATH BUILDTREE_ONLY ) llvm_update_compile_flags(${target_name}) target_link_libraries(${target_name} PRIVATE - PluginCommon ${llvm_libs} ${OPENMP_PTHREAD_LIB}) + PluginCommon ${OPENMP_PTHREAD_LIB}) target_compile_definitions(${target_name} PRIVATE TARGET_NAME=${lib_name}) target_compile_definitions(${target_name} PRIVATE -- GitLab From d7ce6b4d96c8879f38ba4cb5fdb1cc09d5b129e5 Mon Sep 17 00:00:00 2001 From: Mike Rice Date: Sat, 23 Mar 2024 09:53:29 -0700 Subject: [PATCH 148/306] [clang-tidy] Fix result check after overwriteChangedFiles() (#86360) If any return from overwriteChangedFiles is true some fixes were not applied. --- clang-tools-extra/clang-tidy/ClangTidy.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clang-tools-extra/clang-tidy/ClangTidy.cpp b/clang-tools-extra/clang-tidy/ClangTidy.cpp index 40ac6918faf4..b877ea06dc05 100644 --- a/clang-tools-extra/clang-tidy/ClangTidy.cpp +++ b/clang-tools-extra/clang-tidy/ClangTidy.cpp @@ -233,7 +233,7 @@ public: if (!tooling::applyAllReplacements(Replacements.get(), Rewrite)) { llvm::errs() << "Can't apply replacements for file " << File << "\n"; } - AnyNotWritten &= Rewrite.overwriteChangedFiles(); + AnyNotWritten |= Rewrite.overwriteChangedFiles(); } if (AnyNotWritten) { -- GitLab From 87c7f4a12b2a1b723a78d760761ee473f52c4cee Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Sat, 23 Mar 2024 10:03:09 -0700 Subject: [PATCH 149/306] [MC] Remove unnecessary reversal of relocations. NFC Commit f44db24e1fd948c75c87aea017646f16553d3361 (2015) enabled this simplication. --- llvm/lib/MC/ELFObjectWriter.cpp | 9 +-------- .../lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp | 5 ++--- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/llvm/lib/MC/ELFObjectWriter.cpp b/llvm/lib/MC/ELFObjectWriter.cpp index 3c4d3ab9a508..a6fb6b5c1a4e 100644 --- a/llvm/lib/MC/ELFObjectWriter.cpp +++ b/llvm/lib/MC/ELFObjectWriter.cpp @@ -939,18 +939,11 @@ void ELFWriter::writeRelocations(const MCAssembler &Asm, const MCSectionELF &Sec) { std::vector &Relocs = OWriter.Relocations[&Sec]; - // We record relocations by pushing to the end of a vector. Reverse the vector - // to get the relocations in the order they were created. - // In most cases that is not important, but it can be for special sections - // (.eh_frame) or specific relocations (TLS optimizations on SystemZ). - std::reverse(Relocs.begin(), Relocs.end()); - // Sort the relocation entries. MIPS needs this. OWriter.TargetObjectWriter->sortRelocs(Asm, Relocs); const bool Rela = usesRela(Sec); - for (unsigned i = 0, e = Relocs.size(); i != e; ++i) { - const ELFRelocationEntry &Entry = Relocs[e - i - 1]; + for (const ELFRelocationEntry &Entry : Relocs) { unsigned Index = Entry.Symbol ? Entry.Symbol->getIndex() : 0; if (is64Bit()) { diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp b/llvm/lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp index 181b82f14bfe..4d6a00c14a35 100644 --- a/llvm/lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp +++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp @@ -498,10 +498,9 @@ void MipsELFObjectWriter::sortRelocs(const MCAssembler &Asm, assert(Relocs.size() == Sorted.size() && "Some relocs were not consumed"); - // Overwrite the original vector with the sorted elements. The caller expects - // them in reverse order. + // Overwrite the original vector with the sorted elements. unsigned CopyTo = 0; - for (const auto &R : reverse(Sorted)) + for (const auto &R : Sorted) Relocs[CopyTo++] = R.R; } -- GitLab From 3a63f737e29a2382c8ec26c0d360bd77c01a27a1 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Sat, 23 Mar 2024 10:15:47 -0700 Subject: [PATCH 150/306] [MC] Refactor writeRelocations. NFC MIPS is different and should better off use separate code. --- llvm/lib/MC/ELFObjectWriter.cpp | 60 ++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 27 deletions(-) diff --git a/llvm/lib/MC/ELFObjectWriter.cpp b/llvm/lib/MC/ELFObjectWriter.cpp index a6fb6b5c1a4e..f4c6cbc8dd44 100644 --- a/llvm/lib/MC/ELFObjectWriter.cpp +++ b/llvm/lib/MC/ELFObjectWriter.cpp @@ -943,40 +943,28 @@ void ELFWriter::writeRelocations(const MCAssembler &Asm, OWriter.TargetObjectWriter->sortRelocs(Asm, Relocs); const bool Rela = usesRela(Sec); - for (const ELFRelocationEntry &Entry : Relocs) { - unsigned Index = Entry.Symbol ? Entry.Symbol->getIndex() : 0; - - if (is64Bit()) { - write(Entry.Offset); - if (OWriter.TargetObjectWriter->getEMachine() == ELF::EM_MIPS) { - write(uint32_t(Index)); - + if (OWriter.TargetObjectWriter->getEMachine() == ELF::EM_MIPS) { + for (const ELFRelocationEntry &Entry : Relocs) { + uint32_t Symidx = Entry.Symbol ? Entry.Symbol->getIndex() : 0; + if (is64Bit()) { + write(Entry.Offset); + write(uint32_t(Symidx)); write(OWriter.TargetObjectWriter->getRSsym(Entry.Type)); write(OWriter.TargetObjectWriter->getRType3(Entry.Type)); write(OWriter.TargetObjectWriter->getRType2(Entry.Type)); write(OWriter.TargetObjectWriter->getRType(Entry.Type)); + if (Rela) + write(Entry.Addend); } else { - struct ELF::Elf64_Rela ERE64; - ERE64.setSymbolAndType(Index, Entry.Type); - write(ERE64.r_info); - } - if (Rela) - write(Entry.Addend); - } else { - write(uint32_t(Entry.Offset)); - - struct ELF::Elf32_Rela ERE32; - ERE32.setSymbolAndType(Index, Entry.Type); - write(ERE32.r_info); - - if (Rela) - write(uint32_t(Entry.Addend)); - - if (OWriter.TargetObjectWriter->getEMachine() == ELF::EM_MIPS) { + write(uint32_t(Entry.Offset)); + ELF::Elf32_Rela ERE32; + ERE32.setSymbolAndType(Symidx, Entry.Type); + write(ERE32.r_info); + if (Rela) + write(uint32_t(Entry.Addend)); if (uint32_t RType = OWriter.TargetObjectWriter->getRType2(Entry.Type)) { write(uint32_t(Entry.Offset)); - ERE32.setSymbolAndType(0, RType); write(ERE32.r_info); write(uint32_t(0)); @@ -984,13 +972,31 @@ void ELFWriter::writeRelocations(const MCAssembler &Asm, if (uint32_t RType = OWriter.TargetObjectWriter->getRType3(Entry.Type)) { write(uint32_t(Entry.Offset)); - ERE32.setSymbolAndType(0, RType); write(ERE32.r_info); write(uint32_t(0)); } } } + return; + } + for (const ELFRelocationEntry &Entry : Relocs) { + uint32_t Symidx = Entry.Symbol ? Entry.Symbol->getIndex() : 0; + if (is64Bit()) { + write(Entry.Offset); + ELF::Elf64_Rela ERE; + ERE.setSymbolAndType(Symidx, Entry.Type); + write(ERE.r_info); + if (Rela) + write(Entry.Addend); + } else { + write(uint32_t(Entry.Offset)); + ELF::Elf32_Rela ERE; + ERE.setSymbolAndType(Symidx, Entry.Type); + write(ERE.r_info); + if (Rela) + write(uint32_t(Entry.Addend)); + } } } -- GitLab From 39c8e87717fbc611b9e84f62edf656608ae52e5c Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Sat, 23 Mar 2024 18:43:14 +0100 Subject: [PATCH 151/306] [VPlan] Move recording of Inst->VPValue to VPRecipeBuilder (NFCI). (#84464) Instead of keeping a mapping of Inst->VPValues (of their corresponding recipes) in VPlan's Value2VPValue mapping, keep it in VPRecipeBuilder instead. After recently replacing the last user of this mapping after initial construction, this mapping is only needed for recipe construction (to map IR operands to VPValue operands). By moving the mapping, VPlan's VPValue tracking can be simplified and limited only to live-ins. It also allows removing disableValue2VPValue and associated machinery & asserts. PR: https://github.com/llvm/llvm-project/pull/84464 --- .../Transforms/Vectorize/LoopVectorize.cpp | 47 +++++++------------ .../Transforms/Vectorize/VPRecipeBuilder.h | 35 +++++++------- llvm/lib/Transforms/Vectorize/VPlan.h | 25 ++-------- .../Transforms/Vectorize/VPlanTransforms.cpp | 15 +++--- 4 files changed, 46 insertions(+), 76 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 2163930b02c1..28748142ae92 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -7898,6 +7898,18 @@ void LoopVectorizationPlanner::buildVPlans(ElementCount MinVF, } } +iterator_range>> +VPRecipeBuilder::mapToVPValues(User::op_range Operands) { + std::function Fn = [this](Value *Op) { + if (auto *I = dyn_cast(Op)) { + if (auto *R = Ingredient2Recipe.lookup(I)) + return R->getVPSingleValue(); + } + return Plan.getVPValueOrAddLiveIn(Op); + }; + return map_range(Operands, Fn); +} + VPValue *VPRecipeBuilder::createEdgeMask(BasicBlock *Src, BasicBlock *Dst) { assert(is_contained(predecessors(Dst), Src) && "Invalid edge"); @@ -7922,7 +7934,7 @@ VPValue *VPRecipeBuilder::createEdgeMask(BasicBlock *Src, BasicBlock *Dst) { if (OrigLoop->isLoopExiting(Src)) return EdgeMaskCache[Edge] = SrcMask; - VPValue *EdgeMask = Plan.getVPValueOrAddLiveIn(BI->getCondition()); + VPValue *EdgeMask = getVPValueOrAddLiveIn(BI->getCondition(), Plan); assert(EdgeMask && "No Edge Mask found for condition"); if (BI->getSuccessor(0) != Dst) @@ -8383,7 +8395,7 @@ VPReplicateRecipe *VPRecipeBuilder::handleReplication(Instruction *I, BlockInMask = getBlockInMask(I->getParent()); } - auto *Recipe = new VPReplicateRecipe(I, Plan.mapToVPValues(I->operands()), + auto *Recipe = new VPReplicateRecipe(I, mapToVPValues(I->operands()), IsUniform, BlockInMask); return Recipe; } @@ -8399,10 +8411,6 @@ VPRecipeBuilder::tryToCreateWidenRecipe(Instruction *Instr, if (Phi->getParent() != OrigLoop->getHeader()) return tryToBlend(Phi, Operands); - // Always record recipes for header phis. Later first-order recurrence phis - // can have earlier phis as incoming values. - recordRecipeOf(Phi); - if ((Recipe = tryToOptimizeInductionPHI(Phi, Operands, Range))) return Recipe; @@ -8427,14 +8435,6 @@ VPRecipeBuilder::tryToCreateWidenRecipe(Instruction *Instr, PhiRecipe = new VPFirstOrderRecurrencePHIRecipe(Phi, *StartV); } - // Record the incoming value from the backedge, so we can add the incoming - // value from the backedge after all recipes have been created. - auto *Inc = cast( - Phi->getIncomingValueForBlock(OrigLoop->getLoopLatch())); - auto RecipeIter = Ingredient2Recipe.find(Inc); - if (RecipeIter == Ingredient2Recipe.end()) - recordRecipeOf(Inc); - PhisToFix.push_back(PhiRecipe); return PhiRecipe; } @@ -8522,7 +8522,7 @@ static void addCanonicalIVRecipes(VPlan &Plan, Type *IdxTy, bool HasNUW, // Add exit values to \p Plan. VPLiveOuts are added for each LCSSA phi in the // original exit block. static void addUsersInExitBlock(VPBasicBlock *HeaderVPBB, Loop *OrigLoop, - VPlan &Plan) { + VPRecipeBuilder &Builder, VPlan &Plan) { BasicBlock *ExitBB = OrigLoop->getUniqueExitBlock(); BasicBlock *ExitingBB = OrigLoop->getExitingBlock(); // Only handle single-exit loops with unique exit blocks for now. @@ -8533,7 +8533,7 @@ static void addUsersInExitBlock(VPBasicBlock *HeaderVPBB, Loop *OrigLoop, for (PHINode &ExitPhi : ExitBB->phis()) { Value *IncomingValue = ExitPhi.getIncomingValueForBlock(ExitingBB); - VPValue *V = Plan.getVPValueOrAddLiveIn(IncomingValue); + VPValue *V = Builder.getVPValueOrAddLiveIn(IncomingValue, Plan); Plan.addLiveOut(&ExitPhi, V); } } @@ -8603,9 +8603,6 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) { if (!getDecisionAndClampRange(applyIG, Range)) continue; InterleaveGroups.insert(IG); - for (unsigned i = 0; i < IG->getFactor(); i++) - if (Instruction *Member = IG->getMember(i)) - RecipeBuilder.recordRecipeOf(Member); }; // --------------------------------------------------------------------------- @@ -8647,7 +8644,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) { Operands.push_back(Plan->getVPValueOrAddLiveIn( Phi->getIncomingValueForBlock(OrigLoop->getLoopPreheader()))); } else { - auto OpRange = Plan->mapToVPValues(Instr->operands()); + auto OpRange = RecipeBuilder.mapToVPValues(Instr->operands()); Operands = {OpRange.begin(), OpRange.end()}; } @@ -8662,10 +8659,6 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) { RecipeBuilder.tryToCreateWidenRecipe(Instr, Operands, Range, VPBB); if (!Recipe) Recipe = RecipeBuilder.handleReplication(Instr, Range); - for (auto *Def : Recipe->definedValues()) { - auto *UV = Def->getUnderlyingValue(); - Plan->addVPValue(UV, Def); - } RecipeBuilder.setRecipe(Instr, Recipe); if (isa(Recipe)) { @@ -8697,7 +8690,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) { // and there is nothing to fix from vector loop; phis should have incoming // from scalar loop only. } else - addUsersInExitBlock(HeaderVPBB, OrigLoop, *Plan); + addUsersInExitBlock(HeaderVPBB, OrigLoop, RecipeBuilder, *Plan); assert(isa(Plan->getVectorLoopRegion()) && !Plan->getVectorLoopRegion()->getEntryBasicBlock()->empty() && @@ -8765,10 +8758,6 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) { Plan->getVPValueOrAddLiveIn(StrideV)->replaceAllUsesWith(ConstVPV); } - // From this point onwards, VPlan-to-VPlan transformations may change the plan - // in ways that accessing values using original IR values is incorrect. - Plan->disableValue2VPValue(); - VPlanTransforms::dropPoisonGeneratingRecipes(*Plan, [this](BasicBlock *BB) { return Legal->blockNeedsPredication(BB); }); diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h index 29a395c35731..fbb70f333685 100644 --- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h +++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h @@ -52,9 +52,8 @@ class VPRecipeBuilder { EdgeMaskCacheTy EdgeMaskCache; BlockMaskCacheTy BlockMaskCache; - // VPlan-VPlan transformations support: Hold a mapping from ingredients to - // their recipe. To save on memory, only do so for selected ingredients, - // marked by having a nullptr entry in this map. + // VPlan construction support: Hold a mapping from ingredients to + // their recipe. DenseMap Ingredient2Recipe; /// Cross-iteration reduction & first-order recurrence phis for which we need @@ -117,13 +116,10 @@ public: ArrayRef Operands, VFRange &Range, VPBasicBlock *VPBB); - /// Set the recipe created for given ingredient. This operation is a no-op for - /// ingredients that were not marked using a nullptr entry in the map. + /// Set the recipe created for given ingredient. void setRecipe(Instruction *I, VPRecipeBase *R) { - if (!Ingredient2Recipe.count(I)) - return; - assert(Ingredient2Recipe[I] == nullptr && - "Recipe already set for ingredient"); + assert(!Ingredient2Recipe.contains(I) && + "Cannot reset recipe for instruction."); Ingredient2Recipe[I] = R; } @@ -146,14 +142,6 @@ public: /// between SRC and DST. VPValue *getEdgeMask(BasicBlock *Src, BasicBlock *Dst) const; - /// Mark given ingredient for recording its recipe once one is created for - /// it. - void recordRecipeOf(Instruction *I) { - assert((!Ingredient2Recipe.count(I) || Ingredient2Recipe[I] == nullptr) && - "Recipe already set for ingredient"); - Ingredient2Recipe[I] = nullptr; - } - /// Return the recipe created for given ingredient. VPRecipeBase *getRecipe(Instruction *I) { assert(Ingredient2Recipe.count(I) && @@ -171,6 +159,19 @@ public: /// Add the incoming values from the backedge to reduction & first-order /// recurrence cross-iteration phis. void fixHeaderPhis(); + + /// Returns a range mapping the values of the range \p Operands to their + /// corresponding VPValues. + iterator_range>> + mapToVPValues(User::op_range Operands); + + VPValue *getVPValueOrAddLiveIn(Value *V, VPlan &Plan) { + if (auto *I = dyn_cast(V)) { + if (auto *R = Ingredient2Recipe.lookup(I)) + return R->getVPSingleValue(); + } + return Plan.getVPValueOrAddLiveIn(V); + } }; } // end namespace llvm diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index d77c7554d50e..22173954f7ce 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -2872,10 +2872,6 @@ class VPlan { /// definitions are VPValues that hold a pointer to their underlying IR. SmallVector VPLiveInsToFree; - /// Indicates whether it is safe use the Value2VPValue mapping or if the - /// mapping cannot be used any longer, because it is stale. - bool Value2VPValueEnabled = true; - /// Values used outside the plan. MapVector LiveOuts; @@ -2954,10 +2950,6 @@ public: /// Returns VF * UF of the vector loop region. VPValue &getVFxUF() { return VFxUF; } - /// Mark the plan to indicate that using Value2VPValue is not safe any - /// longer, because it may be stale. - void disableValue2VPValue() { Value2VPValueEnabled = false; } - void addVF(ElementCount VF) { VFs.insert(VF); } void setVF(ElementCount VF) { @@ -2987,8 +2979,7 @@ public: void setName(const Twine &newName) { Name = newName.str(); } void addVPValue(Value *V, VPValue *VPV) { - assert((Value2VPValueEnabled || VPV->isLiveIn()) && - "Value2VPValue mapping may be out of date!"); + assert(VPV->isLiveIn() && "VPV must be a live-in."); assert(V && "Trying to add a null Value to VPlan"); assert(!Value2VPValue.count(V) && "Value already exists in VPlan"); Value2VPValue[V] = VPV; @@ -2998,8 +2989,8 @@ public: VPValue *getVPValue(Value *V) { assert(V && "Trying to get the VPValue of a null Value"); assert(Value2VPValue.count(V) && "Value does not exist in VPlan"); - assert((Value2VPValueEnabled || Value2VPValue[V]->isLiveIn()) && - "Value2VPValue mapping may be out of date!"); + assert(Value2VPValue[V]->isLiveIn() && + "Only live-ins should be in mapping"); return Value2VPValue[V]; } @@ -3030,16 +3021,6 @@ public: LLVM_DUMP_METHOD void dump() const; #endif - /// Returns a range mapping the values the range \p Operands to their - /// corresponding VPValues. - iterator_range>> - mapToVPValues(User::op_range Operands) { - std::function Fn = [this](Value *Op) { - return getVPValueOrAddLiveIn(Op); - }; - return map_range(Operands, Fn); - } - /// Returns the VPRegionBlock of the vector loop. VPRegionBlock *getVectorLoopRegion() { return cast(getEntry()->getSingleSuccessor()); diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index a91ccefe4b6d..00cec1148528 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -48,15 +48,14 @@ void VPlanTransforms::VPInstructionsToVPRecipes( VPRecipeBase *NewRecipe = nullptr; if (auto *VPPhi = dyn_cast(&Ingredient)) { auto *Phi = cast(VPPhi->getUnderlyingValue()); - if (const auto *II = GetIntOrFpInductionDescriptor(Phi)) { - VPValue *Start = Plan->getVPValueOrAddLiveIn(II->getStartValue()); - VPValue *Step = - vputils::getOrCreateVPValueForSCEVExpr(*Plan, II->getStep(), SE); - NewRecipe = new VPWidenIntOrFpInductionRecipe(Phi, Start, Step, *II); - } else { - Plan->addVPValue(Phi, VPPhi); + const auto *II = GetIntOrFpInductionDescriptor(Phi); + if (!II) continue; - } + + VPValue *Start = Plan->getVPValueOrAddLiveIn(II->getStartValue()); + VPValue *Step = + vputils::getOrCreateVPValueForSCEVExpr(*Plan, II->getStep(), SE); + NewRecipe = new VPWidenIntOrFpInductionRecipe(Phi, Start, Step, *II); } else { assert(isa(&Ingredient) && "only VPInstructions expected here"); -- GitLab From e14c6fa31a303312d7561aa0c7219d687f3f2c75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Sat, 23 Mar 2024 20:26:20 +0100 Subject: [PATCH 152/306] [clang] [cmake] Add cmake module dir before using GetDarwinLinkerVersion (#86386) Move the code adding top-level cmake/Modules directory to CMAKE_MODULE_PATH prior to including `GetDarwinLinkerVersion`, in order to fix standalone builds. Fixes a regression introduced by 3bc71c2abfa00413fd15cf0e5c08af6ec0d4768b. --- clang/CMakeLists.txt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/clang/CMakeLists.txt b/clang/CMakeLists.txt index ee783d52e4a4..284b2af24dda 100644 --- a/clang/CMakeLists.txt +++ b/clang/CMakeLists.txt @@ -13,6 +13,13 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) set(CLANG_BUILT_STANDALONE TRUE) endif() +# Make sure that our source directory is on the current cmake module path so that +# we can include cmake files from this directory. +list(INSERT CMAKE_MODULE_PATH 0 + "${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules" + "${LLVM_COMMON_CMAKE_UTILS}/Modules" + ) + # Must go below project(..) include(GNUInstallDirs) include(GetDarwinLinkerVersion) @@ -141,13 +148,6 @@ if(CLANG_BUILT_STANDALONE) endif() # LLVM_INCLUDE_TESTS endif() # standalone -# Make sure that our source directory is on the current cmake module path so that -# we can include cmake files from this directory. -list(INSERT CMAKE_MODULE_PATH 0 - "${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules" - "${LLVM_COMMON_CMAKE_UTILS}/Modules" - ) - # This allows disabling clang's XML dependency even if LLVM finds libxml2. # By default, clang depends on libxml2 if LLVM does. option(CLANG_ENABLE_LIBXML2 "Whether libclang may depend on libxml2" -- GitLab From a91cd53de34fcde469d5d02e84554a06798ad2b1 Mon Sep 17 00:00:00 2001 From: Amir Ayupov Date: Sat, 23 Mar 2024 16:08:31 -0700 Subject: [PATCH 153/306] [BOLT][NFC] Refactor BAT metadata data structures Hide the implementations of `FuncHashes` and `BBHashMap` classes, getting rid of `at` accessors that could throw an exception. Test Plan: NFC Reviewers: ayermolo, maksfb, dcci, rafaelauler Reviewed By: rafaelauler Pull Request: https://github.com/llvm/llvm-project/pull/86353 --- .../bolt/Profile/BoltAddressTranslation.h | 117 +++++++++++++++--- bolt/lib/Profile/BoltAddressTranslation.cpp | 85 ++++++------- 2 files changed, 137 insertions(+), 65 deletions(-) diff --git a/bolt/include/bolt/Profile/BoltAddressTranslation.h b/bolt/include/bolt/Profile/BoltAddressTranslation.h index d583ce0b76a2..f8c35f8066f7 100644 --- a/bolt/include/bolt/Profile/BoltAddressTranslation.h +++ b/bolt/include/bolt/Profile/BoltAddressTranslation.h @@ -115,17 +115,6 @@ public: /// Save function and basic block hashes used for metadata dump. void saveMetadata(BinaryContext &BC); - /// Returns BB hash by function output address (after BOLT) and basic block - /// input offset. - size_t getBBHash(uint64_t FuncOutputAddress, uint32_t BBInputOffset) const; - - /// Returns BF hash by function output address (after BOLT). - size_t getBFHash(uint64_t OutputAddress) const; - - /// Returns BB index by function output address (after BOLT) and basic block - /// input offset. - unsigned getBBIndex(uint64_t FuncOutputAddress, uint32_t BBInputOffset) const; - /// True if a given \p Address is a function with translation table entry. bool isBATFunction(uint64_t Address) const { return Maps.count(Address); } @@ -135,7 +124,7 @@ private: /// emitted for the start of the BB. More entries may be emitted to cover /// the location of calls or any instruction that may change control flow. void writeEntriesForBB(MapTy &Map, const BinaryBasicBlock &BB, - uint64_t FuncAddress); + uint64_t FuncInputAddress, uint64_t FuncOutputAddress); /// Write the serialized address translation table for a function. template @@ -158,10 +147,6 @@ private: std::map Maps; - /// Map basic block input offset to a basic block index and hash pair. - using BBHashMap = std::unordered_map>; - std::unordered_map> FuncHashes; - /// Map a function to its basic blocks count std::unordered_map NumBasicBlocksMap; @@ -174,6 +159,106 @@ private: /// Identifies the address of a control-flow changing instructions in a /// translation map entry const static uint32_t BRANCHENTRY = 0x1; + +public: + /// Map basic block input offset to a basic block index and hash pair. + class BBHashMapTy { + class EntryTy { + unsigned Index; + size_t Hash; + + public: + unsigned getBBIndex() const { return Index; } + size_t getBBHash() const { return Hash; } + EntryTy(unsigned Index, size_t Hash) : Index(Index), Hash(Hash) {} + }; + + std::unordered_map Map; + const EntryTy &getEntry(uint32_t BBInputOffset) const { + auto It = Map.find(BBInputOffset); + assert(It != Map.end()); + return It->second; + } + + public: + bool isInputBlock(uint32_t InputOffset) const { + return Map.count(InputOffset); + } + + unsigned getBBIndex(uint32_t BBInputOffset) const { + return getEntry(BBInputOffset).getBBIndex(); + } + + size_t getBBHash(uint32_t BBInputOffset) const { + return getEntry(BBInputOffset).getBBHash(); + } + + void addEntry(uint32_t BBInputOffset, unsigned BBIndex, size_t BBHash) { + Map.emplace(BBInputOffset, EntryTy(BBIndex, BBHash)); + } + + size_t getNumBasicBlocks() const { return Map.size(); } + }; + + /// Map function output address to its hash and basic blocks hash map. + class FuncHashesTy { + class EntryTy { + size_t Hash; + BBHashMapTy BBHashMap; + + public: + size_t getBFHash() const { return Hash; } + const BBHashMapTy &getBBHashMap() const { return BBHashMap; } + EntryTy(size_t Hash) : Hash(Hash) {} + }; + + std::unordered_map Map; + const EntryTy &getEntry(uint64_t FuncOutputAddress) const { + auto It = Map.find(FuncOutputAddress); + assert(It != Map.end()); + return It->second; + } + + public: + size_t getBFHash(uint64_t FuncOutputAddress) const { + return getEntry(FuncOutputAddress).getBFHash(); + } + + const BBHashMapTy &getBBHashMap(uint64_t FuncOutputAddress) const { + return getEntry(FuncOutputAddress).getBBHashMap(); + } + + void addEntry(uint64_t FuncOutputAddress, size_t BFHash) { + Map.emplace(FuncOutputAddress, EntryTy(BFHash)); + } + + size_t getNumFunctions() const { return Map.size(); }; + + size_t getNumBasicBlocks() const { + size_t NumBasicBlocks{0}; + for (auto &I : Map) + NumBasicBlocks += I.second.getBBHashMap().getNumBasicBlocks(); + return NumBasicBlocks; + } + }; + + /// Returns BF hash by function output address (after BOLT). + size_t getBFHash(uint64_t FuncOutputAddress) const { + return FuncHashes.getBFHash(FuncOutputAddress); + } + + /// Returns BBHashMap by function output address (after BOLT). + const BBHashMapTy &getBBHashMap(uint64_t FuncOutputAddress) const { + return FuncHashes.getBBHashMap(FuncOutputAddress); + } + + BBHashMapTy &getBBHashMap(uint64_t FuncOutputAddress) { + return const_cast( + std::as_const(*this).getBBHashMap(FuncOutputAddress)); + } + +private: + FuncHashesTy FuncHashes; }; } // namespace bolt diff --git a/bolt/lib/Profile/BoltAddressTranslation.cpp b/bolt/lib/Profile/BoltAddressTranslation.cpp index 31886f4c8025..57d2aff71977 100644 --- a/bolt/lib/Profile/BoltAddressTranslation.cpp +++ b/bolt/lib/Profile/BoltAddressTranslation.cpp @@ -22,12 +22,10 @@ const char *BoltAddressTranslation::SECTION_NAME = ".note.bolt_bat"; void BoltAddressTranslation::writeEntriesForBB(MapTy &Map, const BinaryBasicBlock &BB, - uint64_t FuncAddress) { - uint64_t HotFuncAddress = ColdPartSource.count(FuncAddress) - ? ColdPartSource[FuncAddress] - : FuncAddress; + uint64_t FuncInputAddress, + uint64_t FuncOutputAddress) { const uint64_t BBOutputOffset = - BB.getOutputAddressRange().first - FuncAddress; + BB.getOutputAddressRange().first - FuncOutputAddress; const uint32_t BBInputOffset = BB.getInputOffset(); // Every output BB must track back to an input BB for profile collection @@ -42,11 +40,14 @@ void BoltAddressTranslation::writeEntriesForBB(MapTy &Map, LLVM_DEBUG(dbgs() << "BB " << BB.getName() << "\n"); LLVM_DEBUG(dbgs() << " Key: " << Twine::utohexstr(BBOutputOffset) << " Val: " << Twine::utohexstr(BBInputOffset) << "\n"); - LLVM_DEBUG(dbgs() << formatv(" Hash: {0:x}\n", - getBBHash(HotFuncAddress, BBInputOffset))); - (void)HotFuncAddress; - LLVM_DEBUG(dbgs() << formatv(" Index: {0}\n", - getBBIndex(HotFuncAddress, BBInputOffset))); + // NB: in `writeEntriesForBB` we use the input address because hashes are + // saved early in `saveMetadata` before output addresses are assigned. + const BBHashMapTy &BBHashMap = getBBHashMap(FuncInputAddress); + (void)BBHashMap; + LLVM_DEBUG( + dbgs() << formatv(" Hash: {0:x}\n", BBHashMap.getBBHash(BBInputOffset))); + LLVM_DEBUG( + dbgs() << formatv(" Index: {0}\n", BBHashMap.getBBIndex(BBInputOffset))); // In case of conflicts (same Key mapping to different Vals), the last // update takes precedence. Of course it is not ideal to have conflicts and // those happen when we have an empty BB that either contained only @@ -63,7 +64,7 @@ void BoltAddressTranslation::writeEntriesForBB(MapTy &Map, const auto InputAddress = BB.getFunction()->getAddress() + InputOffset; const auto OutputAddress = IOAddressMap.lookup(InputAddress); assert(OutputAddress && "Unknown instruction address"); - const auto OutputOffset = *OutputAddress - FuncAddress; + const auto OutputOffset = *OutputAddress - FuncOutputAddress; // Is this the first instruction in the BB? No need to duplicate the entry. if (OutputOffset == BBOutputOffset) @@ -99,7 +100,7 @@ void BoltAddressTranslation::write(const BinaryContext &BC, raw_ostream &OS) { MapTy Map; for (const BinaryBasicBlock *const BB : Function.getLayout().getMainFragment()) - writeEntriesForBB(Map, *BB, Function.getOutputAddress()); + writeEntriesForBB(Map, *BB, InputAddress, OutputAddress); Maps.emplace(Function.getOutputAddress(), std::move(Map)); ReverseMap.emplace(OutputAddress, InputAddress); @@ -113,7 +114,7 @@ void BoltAddressTranslation::write(const BinaryContext &BC, raw_ostream &OS) { ColdPartSource.emplace(FF.getAddress(), Function.getOutputAddress()); Map.clear(); for (const BinaryBasicBlock *const BB : FF) - writeEntriesForBB(Map, *BB, FF.getAddress()); + writeEntriesForBB(Map, *BB, InputAddress, FF.getAddress()); Maps.emplace(FF.getAddress(), std::move(Map)); } @@ -125,11 +126,9 @@ void BoltAddressTranslation::write(const BinaryContext &BC, raw_ostream &OS) { writeMaps(Maps, PrevAddress, OS); BC.outs() << "BOLT-INFO: Wrote " << Maps.size() << " BAT maps\n"; - const uint64_t NumBBHashes = std::accumulate( - FuncHashes.begin(), FuncHashes.end(), 0ull, - [](size_t Acc, const auto &B) { return Acc + B.second.second.size(); }); - BC.outs() << "BOLT-INFO: Wrote " << FuncHashes.size() << " function and " - << NumBBHashes << " basic block hashes\n"; + BC.outs() << "BOLT-INFO: Wrote " << FuncHashes.getNumFunctions() + << " function and " << FuncHashes.getNumBasicBlocks() + << " basic block hashes\n"; } APInt BoltAddressTranslation::calculateBranchEntriesBitMask(MapTy &Map, @@ -176,11 +175,10 @@ void BoltAddressTranslation::writeMaps(std::map &Maps, // Only process cold fragments in cold mode, and vice versa. if (Cold != ColdPartSource.count(Address)) continue; - // NB: here we use the input address because hashes are saved early (in - // `saveMetadata`) before output addresses are assigned. + // NB: in `writeMaps` we use the input address because hashes are saved + // early in `saveMetadata` before output addresses are assigned. const uint64_t HotInputAddress = ReverseMap[Cold ? ColdPartSource[Address] : Address]; - std::pair &FuncHashPair = FuncHashes[HotInputAddress]; MapTy &Map = MapEntry.second; const uint32_t NumEntries = Map.size(); LLVM_DEBUG(dbgs() << "Writing " << NumEntries << " entries for 0x" @@ -194,10 +192,11 @@ void BoltAddressTranslation::writeMaps(std::map &Maps, PrevIndex = HotIndex; } else { // Function hash - LLVM_DEBUG(dbgs() << "Hash: " << formatv("{0:x}\n", FuncHashPair.first)); - OS.write(reinterpret_cast(&FuncHashPair.first), 8); + size_t BFHash = getBFHash(HotInputAddress); + LLVM_DEBUG(dbgs() << "Hash: " << formatv("{0:x}\n", BFHash)); + OS.write(reinterpret_cast(&BFHash), 8); // Number of basic blocks - size_t NumBasicBlocks = FuncHashPair.second.size(); + size_t NumBasicBlocks = getBBHashMap(HotInputAddress).getNumBasicBlocks(); LLVM_DEBUG(dbgs() << "Basic blocks: " << NumBasicBlocks << '\n'); encodeULEB128(NumBasicBlocks, OS); } @@ -221,6 +220,7 @@ void BoltAddressTranslation::writeMaps(std::map &Maps, }); } } + const BBHashMapTy &BBHashMap = getBBHashMap(HotInputAddress); size_t Index = 0; uint64_t InOffset = 0; size_t PrevBBIndex = 0; @@ -233,9 +233,9 @@ void BoltAddressTranslation::writeMaps(std::map &Maps, encodeSLEB128(KeyVal.second - InOffset, OS); InOffset = KeyVal.second; // Keeping InOffset as if BRANCHENTRY is encoded if ((InOffset & BRANCHENTRY) == 0) { - unsigned BBIndex; - size_t BBHash; - std::tie(BBIndex, BBHash) = FuncHashPair.second[InOffset >> 1]; + const bool IsBlock = BBHashMap.isInputBlock(InOffset >> 1); + unsigned BBIndex = IsBlock ? BBHashMap.getBBIndex(InOffset >> 1) : 0; + size_t BBHash = IsBlock ? BBHashMap.getBBHash(InOffset >> 1) : 0; OS.write(reinterpret_cast(&BBHash), 8); // Basic block index in the input binary encodeULEB128(BBIndex - PrevBBIndex, OS); @@ -295,7 +295,7 @@ void BoltAddressTranslation::parseMaps(std::vector &HotFuncs, HotFuncs.push_back(Address); // Function hash const size_t FuncHash = DE.getU64(&Offset, &Err); - FuncHashes[Address].first = FuncHash; + FuncHashes.addEntry(Address, FuncHash); LLVM_DEBUG(dbgs() << formatv("{0:x}: hash {1:x}\n", Address, FuncHash)); // Number of basic blocks const size_t NumBasicBlocks = DE.getULEB128(&Offset, &Err); @@ -355,8 +355,7 @@ void BoltAddressTranslation::parseMaps(std::vector &HotFuncs, BBIndexDelta = DE.getULEB128(&Offset, &Err); BBIndex += BBIndexDelta; // Map basic block hash to hot fragment by input offset - FuncHashes[HotAddress].second.emplace(InputOffset >> 1, - std::pair(BBIndex, BBHash)); + getBBHashMap(HotAddress).addEntry(InputOffset >> 1, BBIndex, BBHash); } LLVM_DEBUG({ dbgs() << formatv( @@ -385,6 +384,8 @@ void BoltAddressTranslation::dump(raw_ostream &OS) { OS << formatv(", hash: {0:x}", getBFHash(Address)); OS << "\n"; OS << "BB mappings:\n"; + const BBHashMapTy &BBHashMap = + getBBHashMap(HotAddress ? HotAddress : Address); for (const auto &Entry : MapEntry.second) { const bool IsBranch = Entry.second & BRANCHENTRY; const uint32_t Val = Entry.second >> 1; // dropping BRANCHENTRY bit @@ -393,8 +394,7 @@ void BoltAddressTranslation::dump(raw_ostream &OS) { if (IsBranch) OS << " (branch)"; else - OS << formatv(" hash: {0:x}", - getBBHash(HotAddress ? HotAddress : Address, Val)); + OS << formatv(" hash: {0:x}", BBHashMap.getBBHash(Val)); OS << "\n"; } OS << "\n"; @@ -515,27 +515,14 @@ void BoltAddressTranslation::saveMetadata(BinaryContext &BC) { if (BF.isIgnored() || (!BC.HasRelocations && !BF.isSimple())) continue; // Prepare function and block hashes - FuncHashes[BF.getAddress()].first = BF.computeHash(); + FuncHashes.addEntry(BF.getAddress(), BF.computeHash()); BF.computeBlockHashes(); + BBHashMapTy &BBHashMap = getBBHashMap(BF.getAddress()); + // Set BF/BB metadata for (const BinaryBasicBlock &BB : BF) - FuncHashes[BF.getAddress()].second.emplace( - BB.getInputOffset(), std::pair(BB.getIndex(), BB.getHash())); + BBHashMap.addEntry(BB.getInputOffset(), BB.getIndex(), BB.getHash()); } } -unsigned BoltAddressTranslation::getBBIndex(uint64_t FuncOutputAddress, - uint32_t BBInputOffset) const { - return FuncHashes.at(FuncOutputAddress).second.at(BBInputOffset).first; -} - -size_t BoltAddressTranslation::getBBHash(uint64_t FuncOutputAddress, - uint32_t BBInputOffset) const { - return FuncHashes.at(FuncOutputAddress).second.at(BBInputOffset).second; -} - -size_t BoltAddressTranslation::getBFHash(uint64_t OutputAddress) const { - return FuncHashes.at(OutputAddress).first; -} - } // namespace bolt } // namespace llvm -- GitLab From 90a7fc366ad05a7dd0465730c055af57c80d62d9 Mon Sep 17 00:00:00 2001 From: "Felix (Ting Wang)" Date: Sun, 24 Mar 2024 08:46:45 +0800 Subject: [PATCH 154/306] [PowerPC][NFC] Add base test case for small-local-dynamic-tls on AIX (#84711) --- ...aix-small-local-dynamic-tls-largeaccess.ll | 632 ++++++++++ .../aix-small-local-dynamic-tls-types.ll | 1066 +++++++++++++++++ 2 files changed, 1698 insertions(+) create mode 100644 llvm/test/CodeGen/PowerPC/aix-small-local-dynamic-tls-largeaccess.ll create mode 100644 llvm/test/CodeGen/PowerPC/aix-small-local-dynamic-tls-types.ll diff --git a/llvm/test/CodeGen/PowerPC/aix-small-local-dynamic-tls-largeaccess.ll b/llvm/test/CodeGen/PowerPC/aix-small-local-dynamic-tls-largeaccess.ll new file mode 100644 index 000000000000..eb16bae67150 --- /dev/null +++ b/llvm/test/CodeGen/PowerPC/aix-small-local-dynamic-tls-largeaccess.ll @@ -0,0 +1,632 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3 +; RUN: llc -verify-machineinstrs -mcpu=pwr7 -ppc-asm-full-reg-names \ +; RUN: -mtriple powerpc64-ibm-aix-xcoff < %s \ +; RUN: | FileCheck %s --check-prefix=SMALL-LOCAL-DYNAMIC-SMALLCM64 +; RUN: llc -verify-machineinstrs -mcpu=pwr7 -ppc-asm-full-reg-names \ +; RUN: -mtriple powerpc64-ibm-aix-xcoff --code-model=large \ +; RUN: < %s | FileCheck %s \ +; RUN: --check-prefix=SMALL-LOCAL-DYNAMIC-LARGECM64 + +; Test disassembly of object. +; RUN: llc -verify-machineinstrs -mcpu=pwr7 \ +; RUN: -mtriple powerpc64-ibm-aix-xcoff -xcoff-traceback-table=false \ +; RUN: --code-model=large -filetype=obj -o %t.o < %s +; RUN: llvm-objdump -D -r --symbol-description %t.o | FileCheck -D#NFA=2 --check-prefix=DIS %s + +@ElementIntTLSv1 = thread_local(localdynamic) global [8187 x i32] zeroinitializer, align 4 ; Within 32K +@ElementIntTLS2 = thread_local(localdynamic) global [4000 x i32] zeroinitializer, align 4 +@ElementIntTLS3 = thread_local(localdynamic) global [4000 x i32] zeroinitializer, align 4 +@ElementIntTLS4 = thread_local(localdynamic) global [4000 x i32] zeroinitializer, align 4 +@ElementIntTLS5 = thread_local(localdynamic) global [4000 x i32] zeroinitializer, align 4 +@ElementIntTLSv2 = thread_local(localdynamic) global [9000 x i32] zeroinitializer, align 4 ; Beyond 32K + +@ElementLongTLS6 = external thread_local(localdynamic) global [60 x i64], align 8 +@ElementLongTLS2 = thread_local(localdynamic) global [3000 x i64] zeroinitializer, align 8 ; Within 32K +@MyTLSGDVar = thread_local global [800 x i64] zeroinitializer, align 8 +@ElementLongTLS3 = thread_local(localdynamic) global [3000 x i64] zeroinitializer, align 8 +@ElementLongTLS4 = thread_local(localdynamic) global [3000 x i64] zeroinitializer, align 8 +@ElementLongTLS5 = thread_local(localdynamic) global [3000 x i64] zeroinitializer, align 8 +@ElementLongTLS = thread_local(localdynamic) local_unnamed_addr global [7800 x i64] zeroinitializer, align 8 ; Beyond 32K + +declare nonnull ptr @llvm.threadlocal.address.p0(ptr nonnull) #1 + +; All accesses use a "faster" local-dynamic sequence directly off the module handle. +; Exercise PPCXCOFFObjectWriter::getRelocTypeAndSignSize/fixup_ppc_half16. +define signext i32 @test1() { +; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: test1: +; SMALL-LOCAL-DYNAMIC-SMALLCM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r6, L..C1(r2) # target-flags(ppc-tlsld) @ElementIntTLS2 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r7, L..C2(r2) # target-flags(ppc-tlsld) @ElementIntTLS3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r8, L..C3(r2) # target-flags(ppc-tlsld) @ElementIntTLS4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r9, L..C4(r2) # target-flags(ppc-tlsld) @ElementIntTLS5 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r5, L..C5(r2) # target-flags(ppc-tlsld) @ElementIntTLSv1 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r4, 1 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: add r6, r3, r6 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: add r7, r3, r7 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: add r8, r3, r8 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: add r9, r3, r9 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stwux r4, r3, r5 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r4, 4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stw r4, 24(r3) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r3, 2 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stw r3, 320(r6) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r3, 3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stw r3, 324(r7) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r3, 88 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stw r4, 328(r8) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stw r3, 332(r9) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r3, 102 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: test1: +; SMALL-LOCAL-DYNAMIC-LARGECM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r6, L..C1@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r7, L..C2@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C0@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r8, L..C3@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r9, L..C4@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r7, L..C2@l(r7) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r8, L..C3@l(r8) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r9, L..C4@l(r9) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r5, L..C1@l(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r6, L..C5@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r4, 1 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r6, L..C5@l(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: add r7, r3, r7 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: add r8, r3, r8 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: add r9, r3, r9 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: add r6, r3, r6 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stwux r4, r3, r5 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r4, 4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stw r4, 24(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r3, 2 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stw r3, 320(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r3, 3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stw r3, 324(r7) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r3, 88 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stw r4, 328(r8) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stw r3, 332(r9) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r3, 102 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: blr +entry: + %tls1 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ElementIntTLSv1) + store i32 1, ptr %tls1, align 4 + %arrayidx1 = getelementptr inbounds [8187 x i32], ptr %tls1, i64 0, i64 6 + store i32 4, ptr %arrayidx1, align 4 + %tls2 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ElementIntTLS2) + %arrayidx2 = getelementptr inbounds [4000 x i32], ptr %tls2, i64 0, i64 80 + store i32 2, ptr %arrayidx2, align 4 + %tls3 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ElementIntTLS3) + %arrayidx3 = getelementptr inbounds [4000 x i32], ptr %tls3, i64 0, i64 81 + store i32 3, ptr %arrayidx3, align 4 + %tls4 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ElementIntTLS4) + %arrayidx4 = getelementptr inbounds [4000 x i32], ptr %tls4, i64 0, i64 82 + store i32 4, ptr %arrayidx4, align 4 + %tls5 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ElementIntTLS5) + %arrayidx5 = getelementptr inbounds [4000 x i32], ptr %tls5, i64 0, i64 83 + store i32 88, ptr %arrayidx5, align 4 + %load1 = load i32, ptr %tls1, align 4 + %load2 = load i32, ptr %arrayidx1, align 4 + %load3 = load i32, ptr %arrayidx2, align 4 + %load4 = load i32, ptr %arrayidx3, align 4 + %load5 = load i32, ptr %arrayidx4, align 4 + %add = add i32 %load1, 88 + %add6 = add i32 %add, %load2 + %add8 = add i32 %add6, %load3 + %add10 = add i32 %add8, %load4 + %add12 = add i32 %add10, %load5 + ret i32 %add12 +} + +; All accesses use a "faster" local-dynamic sequence directly off the module handle. +; Exercise PPCXCOFFObjectWriter::getRelocTypeAndSignSize/fixup_ppc_half16ds. +define i64 @test2() { +; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: test2: +; SMALL-LOCAL-DYNAMIC-SMALLCM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C6(r2) # target-flags(ppc-tlsld) @ElementLongTLS6 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mr r6, r3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r3, 212 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: add r4, r6, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r3, 424(r4) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C7(r2) # target-flags(ppc-tlsld) @ElementLongTLS2 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r4, 203 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: add r3, r6, r3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r4, 1200(r3) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C8(r2) # target-flags(ppc-tlsgdm) @MyTLSGDVar +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C9(r2) # target-flags(ppc-tlsgd) @MyTLSGDVar +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_addr[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r4, 44 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r4, 440(r3) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C10(r2) # target-flags(ppc-tlsld) @ElementLongTLS3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r4, 6 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: add r3, r6, r3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r4, 2000(r3) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C11(r2) # target-flags(ppc-tlsld) @ElementLongTLS4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r3, 100 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: add r4, r6, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r3, 6800(r4) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C12(r2) # target-flags(ppc-tlsld) @ElementLongTLS5 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r4, 882 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: add r3, r6, r3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r4, 8400(r3) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r3, 1191 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: test2: +; SMALL-LOCAL-DYNAMIC-LARGECM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r7, L..C6@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C0@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r4, 212 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mr r6, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C6@l(r7) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: add r3, r6, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r4, 424(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C7@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r4, 203 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C7@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: add r3, r6, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r4, 1200(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C8@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r4, L..C9@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C8@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r4, L..C9@l(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_addr[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r4, 44 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r4, 440(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C10@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r4, 6 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C10@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: add r3, r6, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r4, 2000(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C11@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r4, 100 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C11@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: add r3, r6, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r4, 6800(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C12@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r4, 882 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C12@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: add r3, r6, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r4, 8400(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r3, 1191 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: blr +entry: + %tls1 = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @ElementLongTLS6) + %arrayidx = getelementptr inbounds [60 x i64], ptr %tls1, i64 0, i64 53 + store i64 212, ptr %arrayidx, align 8 + %tls2 = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @ElementLongTLS2) + %arrayidx1 = getelementptr inbounds [3000 x i64], ptr %tls2, i64 0, i64 150 + store i64 203, ptr %arrayidx1, align 8 + %tls3 = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @MyTLSGDVar) + %arrayidx2 = getelementptr inbounds [800 x i64], ptr %tls3, i64 0, i64 55 + store i64 44, ptr %arrayidx2, align 8 + %tls4 = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @ElementLongTLS3) + %arrayidx3 = getelementptr inbounds [3000 x i64], ptr %tls4, i64 0, i64 250 + store i64 6, ptr %arrayidx3, align 8 + %tls5 = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @ElementLongTLS4) + %arrayidx4 = getelementptr inbounds [3000 x i64], ptr %tls5, i64 0, i64 850 + store i64 100, ptr %arrayidx4, align 8 + %tls6 = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @ElementLongTLS5) + %arrayidx5 = getelementptr inbounds [3000 x i64], ptr %tls6, i64 0, i64 1050 + store i64 882, ptr %arrayidx5, align 8 + %load1 = load i64, ptr %arrayidx1, align 8 + %load2 = load i64, ptr %arrayidx3, align 8 + %load3 = load i64, ptr %arrayidx4, align 8 + %add = add i64 %load1, 882 + %add9 = add i64 %add, %load2 + %add11 = add i64 %add9, %load3 + ret i64 %add11 +} + +; Example of one access using the regular local-dynamic access from the TOC. +define signext i32 @test3() { +; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: test3: +; SMALL-LOCAL-DYNAMIC-SMALLCM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r6, L..C1(r2) # target-flags(ppc-tlsld) @ElementIntTLS2 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r7, L..C2(r2) # target-flags(ppc-tlsld) @ElementIntTLS3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r8, L..C3(r2) # target-flags(ppc-tlsld) @ElementIntTLS4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r9, L..C4(r2) # target-flags(ppc-tlsld) @ElementIntTLS5 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r5, L..C13(r2) # target-flags(ppc-tlsld) @ElementIntTLSv2 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r4, 1 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: add r6, r3, r6 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: add r7, r3, r7 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: add r8, r3, r8 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: add r9, r3, r9 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stwux r4, r3, r5 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r4, 4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stw r4, 24(r3) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r3, 2 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stw r3, 320(r6) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r3, 3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stw r3, 324(r7) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r3, 88 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stw r4, 328(r8) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stw r3, 332(r9) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: li r3, 102 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: test3: +; SMALL-LOCAL-DYNAMIC-LARGECM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r6, L..C13@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r7, L..C2@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C0@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r8, L..C3@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r9, L..C4@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r7, L..C2@l(r7) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r8, L..C3@l(r8) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r9, L..C4@l(r9) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r5, L..C13@l(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r6, L..C5@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r4, 1 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r6, L..C5@l(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: add r7, r3, r7 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: add r8, r3, r8 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: add r9, r3, r9 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: add r6, r3, r6 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stwux r4, r3, r5 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r4, 4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stw r4, 24(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r3, 2 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stw r3, 320(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r3, 3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stw r3, 324(r7) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r3, 88 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stw r4, 328(r8) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stw r3, 332(r9) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: li r3, 102 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: blr +entry: + %tls1 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ElementIntTLSv2) + store i32 1, ptr %tls1, align 4 + %arrayidx1 = getelementptr inbounds [9000 x i32], ptr %tls1, i64 0, i64 6 + store i32 4, ptr %arrayidx1, align 4 + %tls2 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ElementIntTLS2) + %arrayidx2 = getelementptr inbounds [4000 x i32], ptr %tls2, i64 0, i64 80 + store i32 2, ptr %arrayidx2, align 4 + %tls3 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ElementIntTLS3) + %arrayidx3 = getelementptr inbounds [4000 x i32], ptr %tls3, i64 0, i64 81 + store i32 3, ptr %arrayidx3, align 4 + %tls4 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ElementIntTLS4) + %arrayidx4 = getelementptr inbounds [4000 x i32], ptr %tls4, i64 0, i64 82 + store i32 4, ptr %arrayidx4, align 4 + %tls5 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ElementIntTLS5) + %arrayidx5 = getelementptr inbounds [4000 x i32], ptr %tls5, i64 0, i64 83 + store i32 88, ptr %arrayidx5, align 4 + %load1 = load i32, ptr %tls1, align 4 + %load2 = load i32, ptr %arrayidx1, align 4 + %load3 = load i32, ptr %arrayidx2, align 4 + %load4 = load i32, ptr %arrayidx3, align 4 + %load5 = load i32, ptr %arrayidx4, align 4 + %add = add i32 %load1, 88 + %add9 = add i32 %add, %load2 + %add11 = add i32 %add9, %load3 + %add13 = add i32 %add11, %load4 + %add15 = add i32 %add13, %load5 + ret i32 %add15 +} + +; DIS: file format aix5coff64-rs6000 +; DIS: Disassembly of section .text: +; DIS: 0000000000000000 (idx: [[#NFA+9]]) .test1: +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} mflr 0 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} stdu 1, -48(1) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 3, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+23]]) _$TLSML[TC] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 6, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+25]]) ElementIntTLSv1[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} std 0, 64(1) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 7, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+27]]) ElementIntTLS3[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 3, 0(3) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+23]]) _$TLSML[TC] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 8, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+29]]) ElementIntTLS4[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 9, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+31]]) ElementIntTLS5[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 7, 16(7) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+27]]) ElementIntTLS3[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 8, 24(8) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+29]]) ElementIntTLS4[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 9, 32(9) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+31]]) ElementIntTLS5[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} bla 0x0 +; DIS-NEXT: {{0*}}[[#ADDR]]: R_RBA (idx: [[#NFA+1]]) .__tls_get_mod[PR] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 5, 8(6) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+25]]) ElementIntTLSv1[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 6, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+33]]) ElementIntTLS2[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 4, 1 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 6, 40(6) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+33]]) ElementIntTLS2[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} add 7, 3, 7 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} add 8, 3, 8 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} add 9, 3, 9 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} add 6, 3, 6 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} stwux 4, 3, 5 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 4, 4 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} stw 4, 24(3) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 3, 2 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} stw 3, 320(6) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 3, 3 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} stw 3, 324(7) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 3, 88 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} stw 4, 328(8) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} stw 3, 332(9) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 3, 102 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addi 1, 1, 48 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 0, 16(1) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} mtlr 0 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} blr + +; DIS: 0000000000000090 (idx: [[#NFA+11]]) .test2: +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} mflr 0 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} stdu 1, -48(1) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 3, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+23]]) _$TLSML[TC] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} std 0, 64(1) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 7, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+35]]) ElementLongTLS6[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 3, 0(3) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+23]]) _$TLSML[TC] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} bla 0x0 +; DIS-NEXT: {{0*}}[[#ADDR]]: R_RBA (idx: [[#NFA+1]]) .__tls_get_mod[PR] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 4, 212 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} mr 6, 3 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 3, 48(7) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+35]]) ElementLongTLS6[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} add 3, 6, 3 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} std 4, 424(3) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 3, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+37]]) ElementLongTLS2[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 4, 203 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 3, 56(3) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+37]]) ElementLongTLS2[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} add 3, 6, 3 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} std 4, 1200(3) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 3, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+39]]) .MyTLSGDVar[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 4, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+41]]) MyTLSGDVar[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 3, 64(3) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+39]]) .MyTLSGDVar[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 4, 72(4) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+41]]) MyTLSGDVar[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} bla 0x0 +; DIS-NEXT: {{0*}}[[#ADDR]]: R_RBA (idx: [[#NFA+3]]) .__tls_get_addr[PR] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 4, 44 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} std 4, 440(3) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 3, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+43]]) ElementLongTLS3[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 4, 6 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 3, 80(3) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+43]]) ElementLongTLS3[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} add 3, 6, 3 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} std 4, 2000(3) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 3, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+45]]) ElementLongTLS4[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 4, 100 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 3, 88(3) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+45]]) ElementLongTLS4[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} add 3, 6, 3 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} std 4, 6800(3) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 3, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+47]]) ElementLongTLS5[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 4, 882 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 3, 96(3) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+47]]) ElementLongTLS5[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} add 3, 6, 3 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} std 4, 8400(3) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 3, 1191 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addi 1, 1, 48 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 0, 16(1) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} mtlr 0 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} blr + +; DIS: 0000000000000140 (idx: [[#NFA+13]]) .test3: +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} mflr 0 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} stdu 1, -48(1) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 3, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+23]]) _$TLSML[TC] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 6, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+49]]) ElementIntTLSv2[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} std 0, 64(1) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 7, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+27]]) ElementIntTLS3[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 3, 0(3) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+23]]) _$TLSML[TC] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 8, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+29]]) ElementIntTLS4[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 9, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+31]]) ElementIntTLS5[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 7, 16(7) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+27]]) ElementIntTLS3[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 8, 24(8) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+29]]) ElementIntTLS4[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 9, 32(9) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+31]]) ElementIntTLS5[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} bla 0x0 +; DIS-NEXT: {{0*}}[[#ADDR]]: R_RBA (idx: [[#NFA+1]]) .__tls_get_mod[PR] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 5, 104(6) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+49]]) ElementIntTLSv2[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addis 6, 2, 0 +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU (idx: [[#NFA+33]]) ElementIntTLS2[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 4, 1 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 6, 40(6) +; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL (idx: [[#NFA+33]]) ElementIntTLS2[TE] +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} add 7, 3, 7 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} add 8, 3, 8 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} add 9, 3, 9 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} add 6, 3, 6 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} stwux 4, 3, 5 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 4, 4 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} stw 4, 24(3) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 3, 2 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} stw 3, 320(6) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 3, 3 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} stw 3, 324(7) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 3, 88 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} stw 4, 328(8) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} stw 3, 332(9) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} li 3, 102 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} addi 1, 1, 48 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} ld 0, 16(1) +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} mtlr 0 +; DIS-NEXT: [[#%x, ADDR:]]: {{.*}} blr + +; DIS: Disassembly of section .data: + +; DIS: 00000000000001d0 (idx: 17) test1[DS]: +; DIS-NEXT: 1d0: 00 00 00 00 +; DIS-NEXT: 00000000000001d0: R_POS (idx: [[#NFA+9]]) .test1 +; DIS-NEXT: 1d4: 00 00 00 00 +; DIS-NEXT: 1d8: 00 00 00 00 +; DIS-NEXT: 00000000000001d8: R_POS (idx: [[#NFA+21]]) TOC[TC0] +; DIS-NEXT: 1dc: 00 00 02 18 + +; DIS: 00000000000001e8 (idx: 19) test2[DS]: +; DIS-NEXT: 1e8: 00 00 00 00 +; DIS-NEXT: 00000000000001e8: R_POS (idx: [[#NFA+11]]) .test2 +; DIS-NEXT: 1ec: 00 00 00 90 +; DIS-NEXT: 1f0: 00 00 00 00 +; DIS-NEXT: 00000000000001f0: R_POS (idx: [[#NFA+21]]) TOC[TC0] +; DIS-NEXT: 1f4: 00 00 02 18 + +; DIS: 0000000000000200 (idx: 21) test3[DS]: +; DIS-NEXT: 200: 00 00 00 00 +; DIS-NEXT: 0000000000000200: R_POS (idx: [[#NFA+13]]) .test3 +; DIS-NEXT: 204: 00 00 01 40 +; DIS-NEXT: 208: 00 00 00 00 +; DIS-NEXT: 0000000000000208: R_POS (idx: [[#NFA+21]]) TOC[TC0] +; DIS-NEXT: 20c: 00 00 02 18 + +; DIS: 0000000000000218 (idx: 25) _$TLSML[TC]: +; DIS-NEXT: 218: 00 00 00 00 +; DIS-NEXT: 0000000000000218: R_TLSML (idx: [[#NFA+23]]) _$TLSML[TC] +; DIS-NEXT: 21c: 00 00 00 00 + +; DIS: 0000000000000220 (idx: 27) ElementIntTLSv1[TE]: +; DIS-NEXT: 220: 00 00 00 00 +; DIS-NEXT: 0000000000000220: R_TLS_LD (idx: [[#NFA+51]]) ElementIntTLSv1[TL] +; DIS-NEXT: 224: 00 00 00 00 + +; DIS: 0000000000000228 (idx: 29) ElementIntTLS3[TE]: +; DIS-NEXT: 228: 00 00 00 00 +; DIS-NEXT: 0000000000000228: R_TLS_LD (idx: [[#NFA+55]]) ElementIntTLS3[TL] +; DIS-NEXT: 22c: 00 00 be 6c + +; DIS: 0000000000000230 (idx: 31) ElementIntTLS4[TE]: +; DIS-NEXT: 230: 00 00 00 00 +; DIS-NEXT: 0000000000000230: R_TLS_LD (idx: [[#NFA+57]]) ElementIntTLS4[TL] +; DIS-NEXT: 234: 00 00 fc ec + +; DIS: 0000000000000238 (idx: 33) ElementIntTLS5[TE]: +; DIS-NEXT: 238: 00 00 00 00 +; DIS-NEXT: 0000000000000238: R_TLS_LD (idx: [[#NFA+59]]) ElementIntTLS5[TL] +; DIS-NEXT: 23c: 00 01 3b 6c + +; DIS: 0000000000000240 (idx: 35) ElementIntTLS2[TE]: +; DIS-NEXT: 240: 00 00 00 00 +; DIS-NEXT: 0000000000000240: R_TLS_LD (idx: [[#NFA+53]]) ElementIntTLS2[TL] +; DIS-NEXT: 244: 00 00 7f ec + +; DIS: 0000000000000248 (idx: 37) ElementLongTLS6[TE]: +; DIS-NEXT: 248: 00 00 00 00 +; DIS-NEXT: 0000000000000248: R_TLS_LD (idx: [[#NFA+5]]) ElementLongTLS6[UL] +; DIS-NEXT: 24c: 00 00 00 00 + +; DIS: 0000000000000250 (idx: 39) ElementLongTLS2[TE]: +; DIS-NEXT: 250: 00 00 00 00 +; DIS-NEXT: 0000000000000250: R_TLS_LD (idx: [[#NFA+63]]) ElementLongTLS2[TL] +; DIS-NEXT: 254: 00 02 06 90 + +; DIS: 0000000000000258 (idx: 41) .MyTLSGDVar[TE]: +; DIS-NEXT: 258: 00 00 00 00 +; DIS-NEXT: 0000000000000258: R_TLSM (idx: [[#NFA+65]]) MyTLSGDVar[TL] +; DIS-NEXT: 25c: 00 00 00 00 + +; DIS: 0000000000000260 (idx: 43) MyTLSGDVar[TE]: +; DIS-NEXT: 260: 00 00 00 00 +; DIS-NEXT: 0000000000000260: R_TLS (idx: [[#NFA+65]]) MyTLSGDVar[TL] +; DIS-NEXT: 264: 00 02 64 50 + +; DIS: 0000000000000268 (idx: 45) ElementLongTLS3[TE]: +; DIS-NEXT: 268: 00 00 00 00 +; DIS-NEXT: 0000000000000268: R_TLS_LD (idx: [[#NFA+67]]) ElementLongTLS3[TL] +; DIS-NEXT: 26c: 00 02 7d 50 + +; DIS: 0000000000000270 (idx: 47) ElementLongTLS4[TE]: +; DIS-NEXT: 270: 00 00 00 00 +; DIS-NEXT: 0000000000000270: R_TLS_LD (idx: [[#NFA+69]]) ElementLongTLS4[TL] +; DIS-NEXT: 274: 00 02 db 10 + +; DIS: 0000000000000278 (idx: 49) ElementLongTLS5[TE]: +; DIS-NEXT: 278: 00 00 00 00 +; DIS-NEXT: 0000000000000278: R_TLS_LD (idx: [[#NFA+71]]) ElementLongTLS5[TL] +; DIS-NEXT: 27c: 00 03 38 d0 + +; DIS: 0000000000000280 (idx: 51) ElementIntTLSv2[TE]: +; DIS-NEXT: 280: 00 00 00 00 +; DIS-NEXT: 0000000000000280: R_TLS_LD (idx: [[#NFA+61]]) ElementIntTLSv2[TL] +; DIS-NEXT: 284: 00 01 79 ec + +; DIS: Disassembly of section .tdata: +; DIS: 0000000000000000 (idx: [[#NFA+51]]) ElementIntTLSv1[TL]: +; DIS: 0000000000007fec (idx: [[#NFA+53]]) ElementIntTLS2[TL]: +; DIS: 000000000000be6c (idx: [[#NFA+55]]) ElementIntTLS3[TL]: +; DIS: 000000000000fcec (idx: [[#NFA+57]]) ElementIntTLS4[TL]: +; DIS: 0000000000013b6c (idx: [[#NFA+59]]) ElementIntTLS5[TL]: +; DIS: 00000000000179ec (idx: [[#NFA+61]]) ElementIntTLSv2[TL]: +; DIS: 0000000000020690 (idx: [[#NFA+63]]) ElementLongTLS2[TL]: +; DIS: 0000000000026450 (idx: [[#NFA+65]]) MyTLSGDVar[TL]: +; DIS: 0000000000027d50 (idx: [[#NFA+67]]) ElementLongTLS3[TL]: +; DIS: 000000000002db10 (idx: [[#NFA+69]]) ElementLongTLS4[TL]: +; DIS: 00000000000338d0 (idx: [[#NFA+71]]) ElementLongTLS5[TL]: +; DIS: 0000000000039690 (idx: [[#NFA+73]]) ElementLongTLS[TL]: diff --git a/llvm/test/CodeGen/PowerPC/aix-small-local-dynamic-tls-types.ll b/llvm/test/CodeGen/PowerPC/aix-small-local-dynamic-tls-types.ll new file mode 100644 index 000000000000..d996d86a23d8 --- /dev/null +++ b/llvm/test/CodeGen/PowerPC/aix-small-local-dynamic-tls-types.ll @@ -0,0 +1,1066 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3 +; RUN: llc -verify-machineinstrs -mcpu=pwr7 -ppc-asm-full-reg-names \ +; RUN: -mtriple powerpc64-ibm-aix-xcoff < %s \ +; RUN: | FileCheck %s --check-prefix=SMALL-LOCAL-DYNAMIC-SMALLCM64 +; RUN: llc -verify-machineinstrs -mcpu=pwr7 -ppc-asm-full-reg-names \ +; RUN: -mtriple powerpc64-ibm-aix-xcoff --code-model=large \ +; RUN: < %s | FileCheck %s \ +; RUN: --check-prefix=SMALL-LOCAL-DYNAMIC-LARGECM64 +; RUN: llc -O0 -verify-machineinstrs -mcpu=pwr7 -ppc-asm-full-reg-names \ +; RUN: -mtriple powerpc64-ibm-aix-xcoff < %s \ +; RUN: | FileCheck %s --check-prefix=SMALL-LOCAL-DYNAMIC-SMALLCM64-O0 +; RUN: llc -O0 -verify-machineinstrs -mcpu=pwr7 -ppc-asm-full-reg-names \ +; RUN: -mtriple powerpc64-ibm-aix-xcoff --code-model=large \ +; RUN: < %s | FileCheck %s \ +; RUN: --check-prefix=SMALL-LOCAL-DYNAMIC-LARGECM64-O0 + +declare nonnull ptr @llvm.threadlocal.address.p0(ptr nonnull) #1 +@tlv_int_init = local_unnamed_addr global i32 87, align 4 + +@tlv_char = thread_local(localdynamic) global i8 1, align 1 +@tlv_short = thread_local(localdynamic) global i8 1, align 2 +@tlv_int = thread_local(localdynamic) global i32 1, align 4 +@internal_tlv_int = internal thread_local(localdynamic) global i32 1, align 4 +@tlv_long = thread_local(localdynamic) global i64 1, align 8 +@internal_tlv_long = internal thread_local(localdynamic) global i64 1, align 8 +@tlv_float = thread_local(localdynamic) global float 1.000000e+00, align 4 +@internal_tlv_double = internal thread_local(localdynamic) global double 1.000000e+00, align 8 + +%struct.anon = type { i32 } +@ThreadLocalStruct = thread_local(localdynamic) global %struct.anon zeroinitializer, align 1 +@a = thread_local(localdynamic) global [87 x i32] zeroinitializer, align 4 + +define nonnull ptr @AddrTest1() local_unnamed_addr { +; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: AddrTest1: +; SMALL-LOCAL-DYNAMIC-SMALLCM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C1(r2) # target-flags(ppc-tlsld) @a +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r3, r3, 12 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: AddrTest1: +; SMALL-LOCAL-DYNAMIC-LARGECM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r6, L..C1@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C0@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r4, L..C1@l(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r3, r3, 12 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: AddrTest1: +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r4, L..C1(r2) # target-flags(ppc-tlsld) @a +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: addi r3, r3, 12 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: AddrTest1: +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r3, 56(r1) # 8-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C1@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, L..C1@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, 56(r1) # 8-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, L..C0@l(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addi r3, r3, 12 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: blr +entry: + %tlv_addr = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @a) + %arrayidx = getelementptr inbounds [87 x i32], ptr %tlv_addr, i64 0, i64 3 + ret ptr %arrayidx +} + +define signext i32 @testUnaligned() { +; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: testUnaligned: +; SMALL-LOCAL-DYNAMIC-SMALLCM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C2(r2) # target-flags(ppc-tlsld) @ThreadLocalStruct +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: lwax r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: testUnaligned: +; SMALL-LOCAL-DYNAMIC-LARGECM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r6, L..C2@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C0@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r4, L..C2@l(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: lwax r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testUnaligned: +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r4, L..C2(r2) # target-flags(ppc-tlsld) @ThreadLocalStruct +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: lwa r3, 0(r3) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testUnaligned: +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C2@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r3, 56(r1) # 8-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C1@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, L..C1@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, 56(r1) # 8-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, L..C2@l(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: lwa r3, 0(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: blr +entry: + %tlv_addr = call align 1 ptr @llvm.threadlocal.address.p0(ptr align 1 @ThreadLocalStruct) + %x = getelementptr inbounds %struct.anon, ptr %tlv_addr, i32 0, i32 0 + %value = load i32, ptr %x, align 1 + ret i32 %value +} + +define void @testChar(i8 noundef signext %x) { +; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: testChar: +; SMALL-LOCAL-DYNAMIC-SMALLCM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mr r6, r3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C3(r2) # target-flags(ppc-tlsld) @tlv_char +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stbx r6, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: testChar: +; SMALL-LOCAL-DYNAMIC-LARGECM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mr r6, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r7, L..C3@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C0@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r4, L..C3@l(r7) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stbx r6, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testChar: +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: # kill: def $r3 killed $r3 killed $x3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stw r3, 60(r1) # 4-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mr r4, r3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: lwz r3, 60(r1) # 4-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r5, L..C3(r2) # target-flags(ppc-tlsld) @tlv_char +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: add r4, r4, r5 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stb r3, 0(r4) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testChar: +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: # kill: def $r3 killed $r3 killed $x3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stw r3, 60(r1) # 4-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C3@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r3, 48(r1) # 8-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C1@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, L..C1@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r5, 48(r1) # 8-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mr r4, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: lwz r3, 60(r1) # 4-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r5, L..C3@l(r5) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: add r4, r4, r5 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stb r3, 0(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: blr +entry: + %tlv_addr = tail call align 1 ptr @llvm.threadlocal.address.p0(ptr align 1 @tlv_char) + store i8 %x, ptr %tlv_addr, align 1 + ret void +} + +define void @testShort(i16 noundef signext %x) { +; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: testShort: +; SMALL-LOCAL-DYNAMIC-SMALLCM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mr r6, r3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C4(r2) # target-flags(ppc-tlsld) @tlv_short +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: sthx r6, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: testShort: +; SMALL-LOCAL-DYNAMIC-LARGECM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mr r6, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r7, L..C4@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C0@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r4, L..C4@l(r7) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: sthx r6, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testShort: +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: # kill: def $r3 killed $r3 killed $x3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stw r3, 60(r1) # 4-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mr r4, r3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: lwz r3, 60(r1) # 4-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r5, L..C4(r2) # target-flags(ppc-tlsld) @tlv_short +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: add r4, r4, r5 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: sth r3, 0(r4) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testShort: +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: # kill: def $r3 killed $r3 killed $x3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stw r3, 60(r1) # 4-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C4@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r3, 48(r1) # 8-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C1@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, L..C1@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r5, 48(r1) # 8-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mr r4, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: lwz r3, 60(r1) # 4-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r5, L..C4@l(r5) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: add r4, r4, r5 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: sth r3, 0(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: blr +entry: + %tlv_addr = tail call align 2 ptr @llvm.threadlocal.address.p0(ptr align 2 @tlv_short) + store i16 %x, ptr %tlv_addr, align 2 + ret void +} + +define signext i32 @testInt1() { +; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: testInt1: +; SMALL-LOCAL-DYNAMIC-SMALLCM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C5(r2) # target-flags(ppc-tlsld) @tlv_int +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: lwax r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: testInt1: +; SMALL-LOCAL-DYNAMIC-LARGECM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r6, L..C5@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C0@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r4, L..C5@l(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: lwax r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testInt1: +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r4, L..C5(r2) # target-flags(ppc-tlsld) @tlv_int +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: lwa r3, 0(r3) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testInt1: +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C5@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r3, 56(r1) # 8-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C1@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, L..C1@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, 56(r1) # 8-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, L..C5@l(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: lwa r3, 0(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: blr +entry: + %tlv_addr = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @tlv_int) + %value = load i32, ptr %tlv_addr, align 4 + ret i32 %value +} + +define signext i32 @testInt2() { +; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: testInt2: +; SMALL-LOCAL-DYNAMIC-SMALLCM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C6(r2) # target-flags(ppc-tlsld) @internal_tlv_int +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: lwzx r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C7(r2) # @tlv_int_init +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: lwz r4, 0(r4) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: add r3, r4, r3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: extsw r3, r3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: testInt2: +; SMALL-LOCAL-DYNAMIC-LARGECM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r6, L..C6@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C0@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r4, L..C6@l(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: lwzx r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r4, L..C7@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r4, L..C7@l(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: lwz r4, 0(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: add r3, r4, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: extsw r3, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testInt2: +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r4, L..C6(r2) # target-flags(ppc-tlsld) @internal_tlv_int +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: lwz r4, 0(r3) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, L..C7(r2) # @tlv_int_init +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: lwz r3, 0(r3) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: extsw r3, r3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testInt2: +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C6@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r3, 56(r1) # 8-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C1@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, L..C1@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, 56(r1) # 8-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, L..C6@l(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: lwz r4, 0(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C7@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, L..C7@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: lwz r3, 0(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: extsw r3, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: blr +entry: + %tlv_addr = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @internal_tlv_int) + %tlv_val = load i32, ptr %tlv_addr, align 4 + %global_val = load i32, ptr @tlv_int_init, align 4 + %sum = add nsw i32 %global_val, %tlv_val + ret i32 %sum +} + +define signext i64 @testLong1() { +; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: testLong1: +; SMALL-LOCAL-DYNAMIC-SMALLCM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C8(r2) # target-flags(ppc-tlsld) @tlv_long +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ldx r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: testLong1: +; SMALL-LOCAL-DYNAMIC-LARGECM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r6, L..C8@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C0@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r4, L..C8@l(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ldx r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testLong1: +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r4, L..C8(r2) # target-flags(ppc-tlsld) @tlv_long +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, 0(r3) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testLong1: +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C8@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r3, 56(r1) # 8-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C1@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, L..C1@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, 56(r1) # 8-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, L..C8@l(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, 0(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: blr +entry: + %tlv_addr = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @tlv_long) + %value = load i64, ptr %tlv_addr, align 4 + ret i64 %value +} + +define void @testLong2(i64 noundef signext %x) { +; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: testLong2: +; SMALL-LOCAL-DYNAMIC-SMALLCM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C9(r2) # target-flags(ppc-tlsld) @internal_tlv_long +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ldx r5, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r5, r5, 9 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdx r5, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: testLong2: +; SMALL-LOCAL-DYNAMIC-LARGECM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r6, L..C9@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C0@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r4, L..C9@l(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ldx r5, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r5, r5, 9 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdx r5, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testLong2: +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r4, L..C9(r2) # target-flags(ppc-tlsld) @internal_tlv_long +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: add r4, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, 0(r4) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: addi r3, r3, 9 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: std r3, 0(r4) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testLong2: +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C9@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r3, 56(r1) # 8-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C1@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, L..C1@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, 56(r1) # 8-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, L..C9@l(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: add r4, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, 0(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addi r3, r3, 9 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r3, 0(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: blr +entry: + %tlv_addr = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @internal_tlv_long) + %value = load i64, ptr %tlv_addr, align 8 + %add = add nsw i64 %value, 9 + store i64 %add, ptr %tlv_addr, align 8 + ret void +} + +define i32 @testLong3() { +; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: testLong3: +; SMALL-LOCAL-DYNAMIC-SMALLCM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C8(r2) # target-flags(ppc-tlsld) @tlv_long +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ldx r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: testLong3: +; SMALL-LOCAL-DYNAMIC-LARGECM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r6, L..C8@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C0@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r4, L..C8@l(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ldx r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testLong3: +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r4, L..C8(r2) # target-flags(ppc-tlsld) @tlv_long +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, 0(r3) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: # kill: def $r3 killed $r3 killed $x3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: clrldi r3, r3, 32 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testLong3: +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C8@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r3, 56(r1) # 8-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C1@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, L..C1@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, 56(r1) # 8-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, L..C8@l(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, 0(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: # kill: def $r3 killed $r3 killed $x3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: clrldi r3, r3, 32 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: blr +entry: + %tlv_addr = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @tlv_long) + %value = load i64, ptr %tlv_addr, align 8 + %conv = trunc i64 %value to i32 + ret i32 %conv +} + +define void @testFloat1(float noundef %x) { +; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: testFloat1: +; SMALL-LOCAL-DYNAMIC-SMALLCM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: vspltisw v2, 1 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: vspltisw v3, 8 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: xvcvsxwdp vs0, vs34 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C10(r2) # target-flags(ppc-tlsld) @tlv_float +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: lfsx f1, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: fadds f0, f1, f0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: xvcvsxwdp vs1, vs35 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: fadds f0, f0, f1 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stfsx f0, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: testFloat1: +; SMALL-LOCAL-DYNAMIC-LARGECM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: vspltisw v2, 1 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r6, L..C10@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C0@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: vspltisw v3, 8 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: xvcvsxwdp vs0, vs34 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r4, L..C10@l(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: lfsx f1, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: fadds f0, f1, f0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: xvcvsxwdp vs1, vs35 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: fadds f0, f0, f1 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stfsx f0, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testFloat1: +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r4, L..C10(r2) # target-flags(ppc-tlsld) @tlv_float +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: lfs f0, 0(r3) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r4, L..C11(r2) # %const.1 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: lfs f1, 0(r4) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: fadds f0, f0, f1 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r4, L..C12(r2) # %const.0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: lfs f1, 0(r4) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: fadds f0, f0, f1 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stfs f0, 0(r3) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testFloat1: +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C10@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r3, 56(r1) # 8-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C1@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, L..C1@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, 56(r1) # 8-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, L..C10@l(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: lfs f0, 0(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r4, L..C11@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, L..C11@l(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: lfs f1, 0(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: fadds f0, f0, f1 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r4, L..C12@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, L..C12@l(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: lfs f1, 0(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: fadds f0, f0, f1 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stfs f0, 0(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: blr +entry: + %tlv_addr = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @tlv_float) + %value = load float, ptr %tlv_addr, align 4 + %inc = fadd float %value, 1.000000e+00 + %add = fadd float %inc, 8.000000e+00 + store float %add, ptr %tlv_addr, align 4 + ret void +} + +define i32 @testFloat2() { +; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: testFloat2: +; SMALL-LOCAL-DYNAMIC-SMALLCM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C10(r2) # target-flags(ppc-tlsld) @tlv_float +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: lfsx f0, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r3, r1, 60 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: xscvdpsxws f0, f0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stfiwx f0, 0, r3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: lwz r3, 60(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: testFloat2: +; SMALL-LOCAL-DYNAMIC-LARGECM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r6, L..C10@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C0@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r4, L..C10@l(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: lfsx f0, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r3, r1, 60 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: xscvdpsxws f0, f0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stfiwx f0, 0, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: lwz r3, 60(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testFloat2: +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r4, L..C10(r2) # target-flags(ppc-tlsld) @tlv_float +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: lfs f0, 0(r3) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: fctiwz f0, f0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stfd f0, 56(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: lwa r3, 60(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: clrldi r3, r3, 32 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testFloat2: +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C10@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r3, 48(r1) # 8-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C1@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, L..C1@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, 48(r1) # 8-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, L..C10@l(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: lfs f0, 0(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: fctiwz f0, f0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stfd f0, 56(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: lwa r3, 60(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: clrldi r3, r3, 32 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: blr +entry: + %tlv_addr = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @tlv_float) + %value = load float, ptr %tlv_addr, align 4 + %conv = fptosi float %value to i32 + ret i32 %conv +} + +define void @testDouble1(double noundef %x) { +; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: testDouble1: +; SMALL-LOCAL-DYNAMIC-SMALLCM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C11(r2) # target-flags(ppc-tlsld) @internal_tlv_double +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stfdx f1, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: testDouble1: +; SMALL-LOCAL-DYNAMIC-LARGECM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r6, L..C11@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C0@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r4, L..C11@l(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stfdx f1, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testDouble1: +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stdu r1, -48(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: std r0, 64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r4, L..C13(r2) # target-flags(ppc-tlsld) @internal_tlv_double +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stxsdx f1, 0, r3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: addi r1, r1, 48 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testDouble1: +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C13@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r3, 56(r1) # 8-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C1@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, L..C1@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, 56(r1) # 8-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, L..C13@l(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: add r3, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stxsdx f1, 0, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: blr +entry: + %tlv_addr = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @internal_tlv_double) + store double %x, ptr %tlv_addr, align 8 + ret void +} + +define i32 @testDouble2() { +; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: testDouble2: +; SMALL-LOCAL-DYNAMIC-SMALLCM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r4, L..C11(r2) # target-flags(ppc-tlsld) @internal_tlv_double +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: lfdx f0, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r3, r1, 60 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: xscvdpsxws f0, f0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: stfiwx f0, 0, r3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: lwz r3, 60(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: testDouble2: +; SMALL-LOCAL-DYNAMIC-LARGECM64: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r3, L..C0@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addis r6, L..C11@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r3, L..C0@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r4, L..C11@l(r6) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: lfdx f0, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r3, r1, 60 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: xscvdpsxws f0, f0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: stfiwx f0, 0, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: lwz r3, 60(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testDouble2: +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stdu r1, -64(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: std r0, 80(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML" +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r4, L..C13(r2) # target-flags(ppc-tlsld) @internal_tlv_double +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: lfdx f0, r3, r4 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: xscvdpsxws f0, f0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: addi r3, r1, 52 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: stfiwx f0, 0, r3 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: lwz r3, 52(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: clrldi r3, r3, 32 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: addi r1, r1, 64 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT: blr +; +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testDouble2: +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0: # %bb.0: # %entry +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mflr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stdu r1, -80(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r0, 96(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C13@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: std r3, 56(r1) # 8-byte Folded Spill +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addis r3, L..C1@u(r2) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r3, L..C1@l(r3) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: bla .__tls_get_mod[PR] +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, 56(r1) # 8-byte Folded Reload +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r4, L..C13@l(r4) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: lfdx f0, r3, r4 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: xscvdpsxws f0, f0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addi r3, r1, 68 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: stfiwx f0, 0, r3 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: lwz r3, 68(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: clrldi r3, r3, 32 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: addi r1, r1, 80 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: ld r0, 16(r1) +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: mtlr r0 +; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT: blr +entry: + %tlv_addr = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @internal_tlv_double) + %value = load double, ptr %tlv_addr, align 8 + %conv = fptosi double %value to i32 + ret i32 %conv +} -- GitLab From a118769ec8dc94b8332fe50c9240fecc8065a417 Mon Sep 17 00:00:00 2001 From: Nico Weber Date: Sat, 23 Mar 2024 20:56:45 -0400 Subject: [PATCH 155/306] [gn] port 3bc71c2abf --- llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn b/llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn index efb324713cfe..d533e79e6374 100644 --- a/llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn +++ b/llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn @@ -65,6 +65,7 @@ write_cmake_config("lit_common_configured") { "SANITIZER_USE_STATIC_CXX_ABI_PYBOOL=False", "SANITIZER_USE_STATIC_LLVM_UNWINDER_PYBOOL=False", "COMPILER_RT_HAS_AARCH64_SME_PYBOOL=False", + "COMPILER_RT_DARWIN_LINKER_VERSION=", "COMPILER_RT_HAS_LLD_PYBOOL=True", "COMPILER_RT_HAS_GWP_ASAN_PYBOOL=False", "HAVE_RPC_XDR_H=0", -- GitLab From 8e698a1d8e6984e41d2fdf159393d01951d87c21 Mon Sep 17 00:00:00 2001 From: LLVM GN Syncbot Date: Sun, 24 Mar 2024 00:56:59 +0000 Subject: [PATCH 156/306] [gn build] Port b68e2eba0bc8 --- llvm/utils/gn/secondary/libcxx/include/BUILD.gn | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn index 3c8b80526773..ac111f902513 100644 --- a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn +++ b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn @@ -287,6 +287,7 @@ if (current_toolchain == default_toolchain) { "__algorithm/shift_right.h", "__algorithm/shuffle.h", "__algorithm/sift_down.h", + "__algorithm/simd_utils.h", "__algorithm/sort.h", "__algorithm/sort_heap.h", "__algorithm/stable_partition.h", -- GitLab From d7d2f7ca62400ed4a3f8f89062d2aeec61bd29d4 Mon Sep 17 00:00:00 2001 From: Amir Ayupov Date: Sat, 23 Mar 2024 19:11:49 -0700 Subject: [PATCH 157/306] [BOLT] Emit intra-function control flow in YAMLBAT Attach branch counters to YAML profile, covering intra-function control flow. Depends on: https://github.com/llvm/llvm-project/pull/86353 Test Plan: Updated bolt/test/X86/bolt-address-translation-yaml.test Reviewers: rafaelauler, dcci, ayermolo, maksfb Reviewed By: rafaelauler Pull Request: https://github.com/llvm/llvm-project/pull/76911 --- .../bolt/Profile/BoltAddressTranslation.h | 5 ++ bolt/lib/Profile/DataAggregator.cpp | 46 +++++++++++++++++++ .../X86/bolt-address-translation-yaml.test | 14 +++++- 3 files changed, 64 insertions(+), 1 deletion(-) diff --git a/bolt/include/bolt/Profile/BoltAddressTranslation.h b/bolt/include/bolt/Profile/BoltAddressTranslation.h index f8c35f8066f7..51fdadce8085 100644 --- a/bolt/include/bolt/Profile/BoltAddressTranslation.h +++ b/bolt/include/bolt/Profile/BoltAddressTranslation.h @@ -257,6 +257,11 @@ public: std::as_const(*this).getBBHashMap(FuncOutputAddress)); } + /// Returns the number of basic blocks in a function. + size_t getNumBasicBlocks(uint64_t OutputAddress) const { + return NumBasicBlocksMap.at(OutputAddress); + } + private: FuncHashesTy FuncHashes; }; diff --git a/bolt/lib/Profile/DataAggregator.cpp b/bolt/lib/Profile/DataAggregator.cpp index 37c637a44a0e..b0f1cccc3256 100644 --- a/bolt/lib/Profile/DataAggregator.cpp +++ b/bolt/lib/Profile/DataAggregator.cpp @@ -2310,6 +2310,52 @@ std::error_code DataAggregator::writeBATYAML(BinaryContext &BC, BP.Functions.emplace_back( YAMLProfileWriter::convert(Function, /*UseDFS=*/false)); } + + for (const auto &KV : NamesToBranches) { + const StringRef FuncName = KV.first; + const FuncBranchData &Branches = KV.second; + yaml::bolt::BinaryFunctionProfile YamlBF; + BinaryData *BD = BC.getBinaryDataByName(FuncName); + assert(BD); + uint64_t FuncAddress = BD->getAddress(); + if (!BAT->isBATFunction(FuncAddress)) + continue; + // Filter out cold fragments + if (!BD->getSectionName().equals(BC.getMainCodeSectionName())) + continue; + BinaryFunction *BF = BC.getBinaryFunctionAtAddress(FuncAddress); + assert(BF); + YamlBF.Name = FuncName.str(); + YamlBF.Id = BF->getFunctionNumber(); + YamlBF.Hash = BAT->getBFHash(FuncAddress); + YamlBF.ExecCount = BF->getKnownExecutionCount(); + YamlBF.NumBasicBlocks = BAT->getNumBasicBlocks(FuncAddress); + const BoltAddressTranslation::BBHashMapTy &BlockMap = + BAT->getBBHashMap(FuncAddress); + + auto addSuccProfile = [&](yaml::bolt::BinaryBasicBlockProfile &YamlBB, + uint64_t SuccOffset, unsigned SuccDataIdx) { + const llvm::bolt::BranchInfo &BI = Branches.Data.at(SuccDataIdx); + yaml::bolt::SuccessorInfo SI; + SI.Index = BlockMap.getBBIndex(SuccOffset); + SI.Count = BI.Branches; + SI.Mispreds = BI.Mispreds; + YamlBB.Successors.emplace_back(SI); + }; + + for (const auto &[FromOffset, SuccKV] : Branches.IntraIndex) { + yaml::bolt::BinaryBasicBlockProfile YamlBB; + if (!BlockMap.isInputBlock(FromOffset)) + continue; + YamlBB.Index = BlockMap.getBBIndex(FromOffset); + YamlBB.Hash = BlockMap.getBBHash(FromOffset); + for (const auto &[SuccOffset, SuccDataIdx] : SuccKV) + addSuccProfile(YamlBB, SuccOffset, SuccDataIdx); + if (YamlBB.ExecCount || !YamlBB.Successors.empty()) + YamlBF.Blocks.emplace_back(YamlBB); + } + BP.Functions.emplace_back(YamlBF); + } } // Write the profile. diff --git a/bolt/test/X86/bolt-address-translation-yaml.test b/bolt/test/X86/bolt-address-translation-yaml.test index ee54a90a9f2a..be3bcb2a0c6d 100644 --- a/bolt/test/X86/bolt-address-translation-yaml.test +++ b/bolt/test/X86/bolt-address-translation-yaml.test @@ -25,6 +25,7 @@ READ-BAT-CHECK: BOLT-INFO: Parsed 5 BAT entries READ-BAT-CHECK: PERF2BOLT: read 79 aggregated LBR entries YAML-BAT-CHECK: functions: +# Function not covered by BAT - has insns in basic block YAML-BAT-CHECK: - name: main YAML-BAT-CHECK-NEXT: fid: 2 YAML-BAT-CHECK-NEXT: hash: 0x9895746D48B2C876 @@ -35,6 +36,17 @@ YAML-BAT-CHECK-NEXT: - bid: 0 YAML-BAT-CHECK-NEXT: insns: 26 YAML-BAT-CHECK-NEXT: hash: 0xA900AE79CFD40000 YAML-BAT-CHECK-NEXT: succ: [ { bid: 3, cnt: 0 }, { bid: 1, cnt: 0 } ] +# Function covered by BAT - doesn't have insns in basic block +YAML-BAT-CHECK: - name: usqrt +YAML-BAT-CHECK-NEXT: fid: [[#]] +YAML-BAT-CHECK-NEXT: hash: 0x99E67ED32A203023 +YAML-BAT-CHECK-NEXT: exec: 21 +YAML-BAT-CHECK-NEXT: nblocks: 5 +YAML-BAT-CHECK-NEXT: blocks: +YAML-BAT-CHECK: - bid: 1 +YAML-BAT-CHECK-NEXT: insns: [[#]] +YAML-BAT-CHECK-NEXT: hash: 0xD70DC695320E0010 +YAML-BAT-CHECK-NEXT: succ: {{.*}} { bid: 2, cnt: [[#]] } CHECK-BOLT-YAML: pre-processing profile using YAML profile reader -CHECK-BOLT-YAML-NEXT: 1 out of 16 functions in the binary (6.2%) have non-empty execution profile +CHECK-BOLT-YAML-NEXT: 5 out of 16 functions in the binary (31.2%) have non-empty execution profile -- GitLab From 7c9b5228da94a44f5e3948814d896de537d162bb Mon Sep 17 00:00:00 2001 From: Owen Anderson Date: Sat, 23 Mar 2024 21:49:29 -0500 Subject: [PATCH 158/306] Only check assertions that were meant to apply to the normal case of non-splat vector SREM expansion when we aren't hitting the special case. (#86238) Fixes https://github.com/llvm/llvm-project/issues/84830 Introduced in https://github.com/llvm/llvm-project/pull/82706 --- llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp | 10 +++++----- llvm/test/CodeGen/AArch64/srem-vec-crash.ll | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 5 deletions(-) create mode 100644 llvm/test/CodeGen/AArch64/srem-vec-crash.ll diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp index da29b1d5b312..8be03b66e155 100644 --- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -6916,6 +6916,11 @@ TargetLowering::prepareSREMEqFold(EVT SETCCVT, SDValue REMNode, // Q = floor((2 * A) / (2^K)) APInt Q = (2 * A).udiv(APInt::getOneBitSet(W, K)); + assert(APInt::getAllOnes(SVT.getSizeInBits()).ugt(A) && + "We are expecting that A is always less than all-ones for SVT"); + assert(APInt::getAllOnes(ShSVT.getSizeInBits()).ugt(K) && + "We are expecting that K is always less than all-ones for ShSVT"); + // If D was a power of two, apply the alternate constant derivation. if (D0.isOne()) { // A = 2^(W-1) @@ -6924,11 +6929,6 @@ TargetLowering::prepareSREMEqFold(EVT SETCCVT, SDValue REMNode, Q = APInt::getAllOnes(W - K).zext(W); } - assert(APInt::getAllOnes(SVT.getSizeInBits()).ugt(A) && - "We are expecting that A is always less than all-ones for SVT"); - assert(APInt::getAllOnes(ShSVT.getSizeInBits()).ugt(K) && - "We are expecting that K is always less than all-ones for ShSVT"); - // If the divisor is 1 the result can be constant-folded. Likewise, we // don't care about INT_MIN lanes, those can be set to undef if appropriate. if (D.isOne()) { diff --git a/llvm/test/CodeGen/AArch64/srem-vec-crash.ll b/llvm/test/CodeGen/AArch64/srem-vec-crash.ll new file mode 100644 index 000000000000..0fce8de30d4d --- /dev/null +++ b/llvm/test/CodeGen/AArch64/srem-vec-crash.ll @@ -0,0 +1,15 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc -mtriple=aarch64-unknown-unknown < %s | FileCheck %s + +define i32 @pr84830(i1 %arg) { +; CHECK-LABEL: pr84830: +; CHECK: // %bb.0: // %bb +; CHECK-NEXT: mov w0, #1 // =0x1 +; CHECK-NEXT: ret +bb: + %new0 = srem i1 %arg, true + %last = zext i1 %new0 to i32 + %i = icmp ne i32 %last, 0 + %i1 = select i1 %i, i32 0, i32 1 + ret i32 %i1 +} -- GitLab From 74799f424063a2d751e0f9ea698db1f4efd0d8b2 Mon Sep 17 00:00:00 2001 From: Kazu Hirata Date: Sat, 23 Mar 2024 19:50:15 -0700 Subject: [PATCH 159/306] [memprof] Add call stack IDs to IndexedAllocationInfo (#85888) The indexed MemProf file has a huge amount of redundancy. In a large internal application, 82% of call stacks, stored in IndexedAllocationInfo::CallStack, are duplicates. We should work toward deduplicating call stacks by referring to them with unique IDs with actual call stacks stored in a separate data structure, much like we refer to memprof::Frame with memprof::FrameId. At the same time, we need to facilitate a graceful transition from the current version of the MemProf format to the next. We should be able to read (but not write) the current version of the MemProf file even after we move onto the next one. With those goals in mind, I propose to have an integer ID next to CallStack in IndexedAllocationInfo to refer to a call stack in a succinct manner. We'll gradually increase the areas of the compiler where IDs and call stacks have one-to-one correspondence and eventually remove the existing CallStack field. This patch adds call stack ID, named CSId, to IndexedAllocationInfo and teaches the raw profile reader to compute unique call stack IDs and store them in the new field. It does not introduce any user of the call stack IDs yet, except in verifyFunctionProfileData. --- llvm/include/llvm/ProfileData/MemProf.h | 23 ++++++++++++++++-- llvm/lib/ProfileData/MemProf.cpp | 25 ++++++++++++++++++++ llvm/lib/ProfileData/RawMemProfReader.cpp | 6 ++++- llvm/unittests/ProfileData/InstrProfTest.cpp | 3 ++- llvm/unittests/ProfileData/MemProfTest.cpp | 7 ++++-- 5 files changed, 58 insertions(+), 6 deletions(-) diff --git a/llvm/include/llvm/ProfileData/MemProf.h b/llvm/include/llvm/ProfileData/MemProf.h index 37c19094bc2a..75ea0e49d453 100644 --- a/llvm/include/llvm/ProfileData/MemProf.h +++ b/llvm/include/llvm/ProfileData/MemProf.h @@ -1,6 +1,7 @@ #ifndef LLVM_PROFILEDATA_MEMPROF_H_ #define LLVM_PROFILEDATA_MEMPROF_H_ +#include "llvm/ADT/MapVector.h" #include "llvm/ADT/STLFunctionalExtras.h" #include "llvm/ADT/SmallVector.h" #include "llvm/IR/GlobalValue.h" @@ -252,18 +253,26 @@ struct Frame { } }; +// A type representing the index into the table of call stacks. +using CallStackId = uint64_t; + // Holds allocation information in a space efficient format where frames are // represented using unique identifiers. struct IndexedAllocationInfo { // The dynamic calling context for the allocation in bottom-up (leaf-to-root) // order. Frame contents are stored out-of-line. + // TODO: Remove once we fully transition to CSId. llvm::SmallVector CallStack; + // Conceptually the same as above. We are going to keep both CallStack and + // CallStackId while we are transitioning from CallStack to CallStackId. + CallStackId CSId = 0; // The statistics obtained from the runtime for the allocation. PortableMemInfoBlock Info; IndexedAllocationInfo() = default; - IndexedAllocationInfo(ArrayRef CS, const MemInfoBlock &MB) - : CallStack(CS.begin(), CS.end()), Info(MB) {} + IndexedAllocationInfo(ArrayRef CS, CallStackId CSId, + const MemInfoBlock &MB) + : CallStack(CS.begin(), CS.end()), CSId(CSId), Info(MB) {} // Returns the size in bytes when this allocation info struct is serialized. size_t serializedSize() const { @@ -622,6 +631,16 @@ public: return Frame::deserialize(D); } }; + +// Compute a CallStackId for a given call stack. +CallStackId hashCallStack(ArrayRef CS); + +// Verify that each CallStackId is computed with hashCallStack. This function +// is intended to help transition from CallStack to CSId in +// IndexedAllocationInfo. +void verifyFunctionProfileData( + const llvm::MapVector + &FunctionProfileData); } // namespace memprof } // namespace llvm diff --git a/llvm/lib/ProfileData/MemProf.cpp b/llvm/lib/ProfileData/MemProf.cpp index 0461f0e9f840..bffa4ed19be8 100644 --- a/llvm/lib/ProfileData/MemProf.cpp +++ b/llvm/lib/ProfileData/MemProf.cpp @@ -3,8 +3,10 @@ #include "llvm/IR/Function.h" #include "llvm/ProfileData/InstrProf.h" #include "llvm/ProfileData/SampleProf.h" +#include "llvm/Support/BLAKE3.h" #include "llvm/Support/Endian.h" #include "llvm/Support/EndianStream.h" +#include "llvm/Support/HashBuilder.h" namespace llvm { namespace memprof { @@ -117,5 +119,28 @@ Expected readMemProfSchema(const unsigned char *&Buffer) { return Result; } +CallStackId hashCallStack(ArrayRef CS) { + llvm::HashBuilder, llvm::endianness::little> + HashBuilder; + for (FrameId F : CS) + HashBuilder.add(F); + llvm::BLAKE3Result<8> Hash = HashBuilder.final(); + CallStackId CSId; + std::memcpy(&CSId, Hash.data(), sizeof(Hash)); + return CSId; +} + +void verifyFunctionProfileData( + const llvm::MapVector + &FunctionProfileData) { + for (const auto &[GUID, Record] : FunctionProfileData) { + (void)GUID; + for (const auto &AS : Record.AllocSites) { + assert(AS.CSId == hashCallStack(AS.CallStack)); + (void)AS; + } + } +} + } // namespace memprof } // namespace llvm diff --git a/llvm/lib/ProfileData/RawMemProfReader.cpp b/llvm/lib/ProfileData/RawMemProfReader.cpp index 60c37c417aa0..5dc1ff897815 100644 --- a/llvm/lib/ProfileData/RawMemProfReader.cpp +++ b/llvm/lib/ProfileData/RawMemProfReader.cpp @@ -446,6 +446,8 @@ Error RawMemProfReader::mapRawProfileToRecords() { Callstack.append(Frames.begin(), Frames.end()); } + CallStackId CSId = hashCallStack(Callstack); + // We attach the memprof record to each function bottom-up including the // first non-inline frame. for (size_t I = 0; /*Break out using the condition below*/; I++) { @@ -453,7 +455,7 @@ Error RawMemProfReader::mapRawProfileToRecords() { auto Result = FunctionProfileData.insert({F.Function, IndexedMemProfRecord()}); IndexedMemProfRecord &Record = Result.first->second; - Record.AllocSites.emplace_back(Callstack, Entry.second); + Record.AllocSites.emplace_back(Callstack, CSId, Entry.second); if (!F.IsInlineFrame) break; @@ -471,6 +473,8 @@ Error RawMemProfReader::mapRawProfileToRecords() { } } + verifyFunctionProfileData(FunctionProfileData); + return Error::success(); } diff --git a/llvm/unittests/ProfileData/InstrProfTest.cpp b/llvm/unittests/ProfileData/InstrProfTest.cpp index cd4552a039b3..c9323420bda7 100644 --- a/llvm/unittests/ProfileData/InstrProfTest.cpp +++ b/llvm/unittests/ProfileData/InstrProfTest.cpp @@ -366,7 +366,8 @@ IndexedMemProfRecord makeRecord( const MemInfoBlock &Block = MemInfoBlock()) { llvm::memprof::IndexedMemProfRecord MR; for (const auto &Frames : AllocFrames) - MR.AllocSites.emplace_back(Frames, Block); + MR.AllocSites.emplace_back(Frames, llvm::memprof::hashCallStack(Frames), + Block); for (const auto &Frames : CallSiteFrames) MR.CallSites.push_back(Frames); return MR; diff --git a/llvm/unittests/ProfileData/MemProfTest.cpp b/llvm/unittests/ProfileData/MemProfTest.cpp index f5e4a4aff2ed..1cca44e9b037 100644 --- a/llvm/unittests/ProfileData/MemProfTest.cpp +++ b/llvm/unittests/ProfileData/MemProfTest.cpp @@ -280,7 +280,8 @@ TEST(MemProf, RecordSerializationRoundTrip) { IndexedMemProfRecord Record; for (const auto &ACS : AllocCallStacks) { // Use the same info block for both allocation sites. - Record.AllocSites.emplace_back(ACS, Info); + Record.AllocSites.emplace_back(ACS, llvm::memprof::hashCallStack(ACS), + Info); } Record.CallSites.assign(CallSites); @@ -376,7 +377,9 @@ TEST(MemProf, BaseMemProfReader) { Block.AllocCount = 1U, Block.TotalAccessDensity = 4, Block.TotalLifetime = 200001; std::array CallStack{F1.hash(), F2.hash()}; - FakeRecord.AllocSites.emplace_back(/*CS=*/CallStack, /*MB=*/Block); + FakeRecord.AllocSites.emplace_back( + /*CS=*/CallStack, /*CSId=*/llvm::memprof::hashCallStack(CallStack), + /*MB=*/Block); ProfData.insert({F1.hash(), FakeRecord}); MemProfReader Reader(FrameIdMap, ProfData); -- GitLab From a45e58af1b381cf3c0374332386b8291ec5310f4 Mon Sep 17 00:00:00 2001 From: Matthias Springer Date: Sun, 24 Mar 2024 12:48:19 +0900 Subject: [PATCH 160/306] [mlir][bufferization] Add `BufferViewFlowOpInterface` (#78718) This commit adds the `BufferViewFlowOpInterface` to the bufferization dialect. This interface can be implemented by ops that operate on buffers to indicate that a buffer op result and/or region entry block argument may be the same buffer as a buffer operand (or a view thereof). This interface is queried by the `BufferViewFlowAnalysis`. The new interface has two interface methods: * `populateDependencies`: Implementations use the provided callback to declare dependencies between operands and op results/region entry block arguments. E.g., for `%r = arith.select %c, %m1, %m2 : memref<5xf32>`, the interface implementation should declare two dependencies: %m1 -> %r and %m2 -> %r. * `mayBeTerminalBuffer`: An SSA value is a terminal buffer if the buffer view flow analysis stops at the specified value. E.g., because the value is a newly allocated buffer or because no further information is available about the origin of the buffer. Ops that implement the `RegionBranchOpInterface` or `BranchOpInterface` do not have to implement the `BufferViewFlowOpInterface`. The buffer dependencies can be inferred from those two interfaces. This commit makes the `BufferViewFlowAnalysis` more accurate. For unknown ops, it conservatively used to declare all combinations of operands and op results/region entry block arguments as dependencies (false positives). This is no longer the case. While the analysis is still a "maybe" analysis with false positives (e.g., when analyzing ops such as `arith.select` or `scf.if` where the taken branch is not known at compile time), results and region entry block arguments of unknown ops are now marked as terminal buffers. This commit addresses a TODO in `BufferViewFlowAnalysis.cpp`: ``` // TODO: We should have an op interface instead of a hard-coded list of // interfaces/ops. ``` It is no longer needed to hard-code ops. --- .../BufferViewFlowOpInterfaceImpl.h | 20 +++++ .../IR/BufferViewFlowOpInterface.h | 27 +++++++ .../IR/BufferViewFlowOpInterface.td | 73 +++++++++++++++++++ .../Dialect/Bufferization/IR/CMakeLists.txt | 1 + .../Transforms/BufferViewFlowAnalysis.h | 6 ++ .../BufferViewFlowOpInterfaceImpl.h | 20 +++++ mlir/include/mlir/InitAllDialects.h | 4 + .../BufferViewFlowOpInterfaceImpl.cpp | 44 +++++++++++ .../Dialect/Arith/Transforms/CMakeLists.txt | 1 + .../IR/BufferViewFlowOpInterface.cpp | 18 +++++ .../Dialect/Bufferization/IR/CMakeLists.txt | 1 + .../Transforms/BufferViewFlowAnalysis.cpp | 72 ++++++++++++++---- .../BufferViewFlowOpInterfaceImpl.cpp | 48 ++++++++++++ .../Dialect/MemRef/Transforms/CMakeLists.txt | 2 + .../llvm-project-overlay/mlir/BUILD.bazel | 40 ++++++++++ 15 files changed, 363 insertions(+), 14 deletions(-) create mode 100644 mlir/include/mlir/Dialect/Arith/Transforms/BufferViewFlowOpInterfaceImpl.h create mode 100644 mlir/include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.h create mode 100644 mlir/include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.td create mode 100644 mlir/include/mlir/Dialect/MemRef/Transforms/BufferViewFlowOpInterfaceImpl.h create mode 100644 mlir/lib/Dialect/Arith/Transforms/BufferViewFlowOpInterfaceImpl.cpp create mode 100644 mlir/lib/Dialect/Bufferization/IR/BufferViewFlowOpInterface.cpp create mode 100644 mlir/lib/Dialect/MemRef/Transforms/BufferViewFlowOpInterfaceImpl.cpp diff --git a/mlir/include/mlir/Dialect/Arith/Transforms/BufferViewFlowOpInterfaceImpl.h b/mlir/include/mlir/Dialect/Arith/Transforms/BufferViewFlowOpInterfaceImpl.h new file mode 100644 index 000000000000..a2b3a9bb655b --- /dev/null +++ b/mlir/include/mlir/Dialect/Arith/Transforms/BufferViewFlowOpInterfaceImpl.h @@ -0,0 +1,20 @@ +//===- BufferViewFlowOpInterfaceImpl.h - Buffer View Analysis ---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef MLIR_DIALECT_ARITH_TRANSFORMS_BUFFERVIEWFLOWOPINTERFACEIMPL_H +#define MLIR_DIALECT_ARITH_TRANSFORMS_BUFFERVIEWFLOWOPINTERFACEIMPL_H + +namespace mlir { +class DialectRegistry; + +namespace arith { +void registerBufferViewFlowOpInterfaceExternalModels(DialectRegistry ®istry); +} // namespace arith +} // namespace mlir + +#endif // MLIR_DIALECT_ARITH_TRANSFORMS_BUFFERVIEWFLOWOPINTERFACEIMPL_H diff --git a/mlir/include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.h b/mlir/include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.h new file mode 100644 index 000000000000..84e67fe72b62 --- /dev/null +++ b/mlir/include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.h @@ -0,0 +1,27 @@ +//===- BufferViewFlowOpInterface.h - Buffer View Flow Analysis --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef MLIR_DIALECT_BUFFERIZATION_IR_BUFFERVIEWFLOWOPINTERFACE_H_ +#define MLIR_DIALECT_BUFFERIZATION_IR_BUFFERVIEWFLOWOPINTERFACE_H_ + +#include "mlir/IR/OpDefinition.h" +#include "mlir/Support/LLVM.h" + +namespace mlir { +class ValueRange; + +namespace bufferization { + +using RegisterDependenciesFn = std::function; + +} // namespace bufferization +} // namespace mlir + +#include "mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.h.inc" + +#endif // MLIR_DIALECT_BUFFERIZATION_IR_BUFFERVIEWFLOWOPINTERFACE_H_ diff --git a/mlir/include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.td b/mlir/include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.td new file mode 100644 index 000000000000..58885d742266 --- /dev/null +++ b/mlir/include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.td @@ -0,0 +1,73 @@ +//===-- BufferViewFlowOpInterface.td - Buffer View Flow ----*- tablegen -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef BUFFER_VIEW_FLOW_OP_INTERFACE +#define BUFFER_VIEW_FLOW_OP_INTERFACE + +include "mlir/IR/OpBase.td" + +def BufferViewFlowOpInterface : + OpInterface<"BufferViewFlowOpInterface"> { + let description = [{ + An op interface for the buffer view flow analysis. This interface describes + buffer dependencies between operands and op results/region entry block + arguments. + }]; + let cppNamespace = "::mlir::bufferization"; + let methods = [ + InterfaceMethod< + /*desc=*/[{ + Populate buffer dependencies between operands and op results/region + entry block arguments. + + Implementations should register dependencies between an operand ("X") + and an op result/region entry block argument ("Y") if Y may depend + on X. Y depends on X if Y and X are the same buffer or if Y is a + subview of X. + + Example: + ``` + %r = arith.select %c, %m1, %m2 : memref<5xf32> + ``` + In the above example, %0 may depend on %m1 or %m2 and a correct + interface implementation should call: + - "registerDependenciesFn(%m1, %r)". + - "registerDependenciesFn(%m2, %r)" + }], + /*retType=*/"void", + /*methodName=*/"populateDependencies", + /*args=*/(ins + "::mlir::bufferization::RegisterDependenciesFn" + :$registerDependenciesFn) + >, + InterfaceMethod< + /*desc=*/[{ + Return "true" if the given value may be a terminal buffer. A buffer + value is "terminal" if it cannot be traced back any further in the + buffer view flow analysis. + + Examples: A buffer could be terminal because: + - it is a newly allocated buffer (e.g., "memref.alloc"), + - or: because there is not enough compile-time information available + to make a definite decision (e.g., "memref.realloc" may reallocate + but we do not know for sure; another example are call ops where we + would have to analyze the body of the callee). + + Implementations can assume that the given SSA value is an OpResult of + this operation or a region entry block argument of this operation. + }], + /*retType=*/"bool", + /*methodName=*/"mayBeTerminalBuffer", + /*args=*/(ins "Value":$value), + /*methodBody=*/"", + /*defaultImplementation=*/"return false;" + >, + ]; +} + +#endif // BUFFER_VIEW_FLOW_OP_INTERFACE diff --git a/mlir/include/mlir/Dialect/Bufferization/IR/CMakeLists.txt b/mlir/include/mlir/Dialect/Bufferization/IR/CMakeLists.txt index 31a553f9a32f..13a5bc370a4f 100644 --- a/mlir/include/mlir/Dialect/Bufferization/IR/CMakeLists.txt +++ b/mlir/include/mlir/Dialect/Bufferization/IR/CMakeLists.txt @@ -3,6 +3,7 @@ add_mlir_doc(BufferizationOps BufferizationOps Dialects/ -gen-dialect-doc) add_mlir_interface(AllocationOpInterface) add_mlir_interface(BufferDeallocationOpInterface) add_mlir_interface(BufferizableOpInterface) +add_mlir_interface(BufferViewFlowOpInterface) set(LLVM_TARGET_DEFINITIONS BufferizationEnums.td) mlir_tablegen(BufferizationEnums.h.inc -gen-enum-decls) diff --git a/mlir/include/mlir/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.h b/mlir/include/mlir/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.h index 24825db69f90..9e43265c5dfe 100644 --- a/mlir/include/mlir/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.h +++ b/mlir/include/mlir/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.h @@ -63,6 +63,9 @@ public: /// results have to be changed. void rename(Value from, Value to); + /// Returns "true" if the given value may be a terminal. + bool mayBeTerminalBuffer(Value value) const; + private: /// This function constructs a mapping from values to its immediate /// dependencies. @@ -70,6 +73,9 @@ private: /// Maps values to all immediate dependencies this value can have. ValueMapT dependencies; + + /// A set of all SSA values that may be terminal buffers. + DenseSet terminals; }; } // namespace mlir diff --git a/mlir/include/mlir/Dialect/MemRef/Transforms/BufferViewFlowOpInterfaceImpl.h b/mlir/include/mlir/Dialect/MemRef/Transforms/BufferViewFlowOpInterfaceImpl.h new file mode 100644 index 000000000000..714518a21e97 --- /dev/null +++ b/mlir/include/mlir/Dialect/MemRef/Transforms/BufferViewFlowOpInterfaceImpl.h @@ -0,0 +1,20 @@ +//===- BufferViewFlowOpInterfaceImpl.h - Buffer View Analysis ---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef MLIR_DIALECT_MEMREF_TRANSFORMS_BUFFERVIEWFLOWOPINTERFACEIMPL_H +#define MLIR_DIALECT_MEMREF_TRANSFORMS_BUFFERVIEWFLOWOPINTERFACEIMPL_H + +namespace mlir { +class DialectRegistry; + +namespace memref { +void registerBufferViewFlowOpInterfaceExternalModels(DialectRegistry ®istry); +} // namespace memref +} // namespace mlir + +#endif // MLIR_DIALECT_MEMREF_TRANSFORMS_BUFFERVIEWFLOWOPINTERFACEIMPL_H diff --git a/mlir/include/mlir/InitAllDialects.h b/mlir/include/mlir/InitAllDialects.h index 9bbf12d13254..c558dc53cc7f 100644 --- a/mlir/include/mlir/InitAllDialects.h +++ b/mlir/include/mlir/InitAllDialects.h @@ -21,6 +21,7 @@ #include "mlir/Dialect/Arith/IR/Arith.h" #include "mlir/Dialect/Arith/IR/ValueBoundsOpInterfaceImpl.h" #include "mlir/Dialect/Arith/Transforms/BufferDeallocationOpInterfaceImpl.h" +#include "mlir/Dialect/Arith/Transforms/BufferViewFlowOpInterfaceImpl.h" #include "mlir/Dialect/Arith/Transforms/BufferizableOpInterfaceImpl.h" #include "mlir/Dialect/ArmNeon/ArmNeonDialect.h" #include "mlir/Dialect/ArmSME/IR/ArmSME.h" @@ -52,6 +53,7 @@ #include "mlir/Dialect/MemRef/IR/MemRefMemorySlot.h" #include "mlir/Dialect/MemRef/IR/ValueBoundsOpInterfaceImpl.h" #include "mlir/Dialect/MemRef/Transforms/AllocationOpInterfaceImpl.h" +#include "mlir/Dialect/MemRef/Transforms/BufferViewFlowOpInterfaceImpl.h" #include "mlir/Dialect/MemRef/Transforms/RuntimeOpVerification.h" #include "mlir/Dialect/Mesh/IR/MeshDialect.h" #include "mlir/Dialect/NVGPU/IR/NVGPUDialect.h" @@ -148,6 +150,7 @@ inline void registerAllDialects(DialectRegistry ®istry) { affine::registerValueBoundsOpInterfaceExternalModels(registry); arith::registerBufferDeallocationOpInterfaceExternalModels(registry); arith::registerBufferizableOpInterfaceExternalModels(registry); + arith::registerBufferViewFlowOpInterfaceExternalModels(registry); arith::registerValueBoundsOpInterfaceExternalModels(registry); bufferization::func_ext::registerBufferizableOpInterfaceExternalModels( registry); @@ -157,6 +160,7 @@ inline void registerAllDialects(DialectRegistry ®istry) { gpu::registerBufferDeallocationOpInterfaceExternalModels(registry); linalg::registerAllDialectInterfaceImplementations(registry); memref::registerAllocationOpInterfaceExternalModels(registry); + memref::registerBufferViewFlowOpInterfaceExternalModels(registry); memref::registerRuntimeVerifiableOpInterfaceExternalModels(registry); memref::registerValueBoundsOpInterfaceExternalModels(registry); memref::registerMemorySlotExternalModels(registry); diff --git a/mlir/lib/Dialect/Arith/Transforms/BufferViewFlowOpInterfaceImpl.cpp b/mlir/lib/Dialect/Arith/Transforms/BufferViewFlowOpInterfaceImpl.cpp new file mode 100644 index 000000000000..9df9df86b64f --- /dev/null +++ b/mlir/lib/Dialect/Arith/Transforms/BufferViewFlowOpInterfaceImpl.cpp @@ -0,0 +1,44 @@ +//===- BufferViewFlowOpInterfaceImpl.cpp - Buffer View Flow Analysis ------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "mlir/Dialect/Arith/Transforms/BufferViewFlowOpInterfaceImpl.h" + +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.h" + +using namespace mlir; +using namespace mlir::bufferization; + +namespace mlir { +namespace arith { +namespace { + +struct SelectOpInterface + : public BufferViewFlowOpInterface::ExternalModel { + void + populateDependencies(Operation *op, + RegisterDependenciesFn registerDependenciesFn) const { + auto selectOp = cast(op); + + // Either one of the true/false value may be selected at runtime. + registerDependenciesFn(selectOp.getTrueValue(), selectOp.getResult()); + registerDependenciesFn(selectOp.getFalseValue(), selectOp.getResult()); + } +}; + +} // namespace +} // namespace arith +} // namespace mlir + +void arith::registerBufferViewFlowOpInterfaceExternalModels( + DialectRegistry ®istry) { + registry.addExtension(+[](MLIRContext *ctx, arith::ArithDialect *dialect) { + SelectOp::attachInterface(*ctx); + }); +} diff --git a/mlir/lib/Dialect/Arith/Transforms/CMakeLists.txt b/mlir/lib/Dialect/Arith/Transforms/CMakeLists.txt index 02240601bcd3..12659eaba1fa 100644 --- a/mlir/lib/Dialect/Arith/Transforms/CMakeLists.txt +++ b/mlir/lib/Dialect/Arith/Transforms/CMakeLists.txt @@ -2,6 +2,7 @@ add_mlir_dialect_library(MLIRArithTransforms BufferDeallocationOpInterfaceImpl.cpp BufferizableOpInterfaceImpl.cpp Bufferize.cpp + BufferViewFlowOpInterfaceImpl.cpp EmulateUnsupportedFloats.cpp EmulateWideInt.cpp EmulateNarrowType.cpp diff --git a/mlir/lib/Dialect/Bufferization/IR/BufferViewFlowOpInterface.cpp b/mlir/lib/Dialect/Bufferization/IR/BufferViewFlowOpInterface.cpp new file mode 100644 index 000000000000..ea726a4bfc3f --- /dev/null +++ b/mlir/lib/Dialect/Bufferization/IR/BufferViewFlowOpInterface.cpp @@ -0,0 +1,18 @@ +//===- BufferViewFlowOpInterface.cpp - Buffer View Flow Analysis ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.h" +#include "mlir/Dialect/Bufferization/IR/Bufferization.h" + +namespace mlir { +namespace bufferization { + +#include "mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.cpp.inc" + +} // namespace bufferization +} // namespace mlir diff --git a/mlir/lib/Dialect/Bufferization/IR/CMakeLists.txt b/mlir/lib/Dialect/Bufferization/IR/CMakeLists.txt index 9895db9d93ce..63dcc1eb233e 100644 --- a/mlir/lib/Dialect/Bufferization/IR/CMakeLists.txt +++ b/mlir/lib/Dialect/Bufferization/IR/CMakeLists.txt @@ -4,6 +4,7 @@ add_mlir_dialect_library(MLIRBufferizationDialect BufferDeallocationOpInterface.cpp BufferizationOps.cpp BufferizationDialect.cpp + BufferViewFlowOpInterface.cpp UnstructuredControlFlow.cpp ADDITIONAL_HEADER_DIRS diff --git a/mlir/lib/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.cpp b/mlir/lib/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.cpp index 88ef1b639fc5..9a36057425f3 100644 --- a/mlir/lib/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.cpp +++ b/mlir/lib/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.cpp @@ -8,12 +8,16 @@ #include "mlir/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.h" +#include "mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.h" +#include "mlir/Interfaces/CallInterfaces.h" #include "mlir/Interfaces/ControlFlowInterfaces.h" +#include "mlir/Interfaces/FunctionInterfaces.h" #include "mlir/Interfaces/ViewLikeInterface.h" #include "llvm/ADT/SetOperations.h" #include "llvm/ADT/SetVector.h" using namespace mlir; +using namespace mlir::bufferization; /// Constructs a new alias analysis using the op provided. BufferViewFlowAnalysis::BufferViewFlowAnalysis(Operation *op) { build(op); } @@ -65,18 +69,44 @@ void BufferViewFlowAnalysis::rename(Value from, Value to) { void BufferViewFlowAnalysis::build(Operation *op) { // Registers all dependencies of the given values. auto registerDependencies = [&](ValueRange values, ValueRange dependencies) { - for (auto [value, dep] : llvm::zip(values, dependencies)) + for (auto [value, dep] : llvm::zip_equal(values, dependencies)) this->dependencies[value].insert(dep); }; + // Mark all buffer results and buffer region entry block arguments of the + // given op as terminals. + auto populateTerminalValues = [&](Operation *op) { + for (Value v : op->getResults()) + if (isa(v.getType())) + this->terminals.insert(v); + for (Region &r : op->getRegions()) + for (BlockArgument v : r.getArguments()) + if (isa(v.getType())) + this->terminals.insert(v); + }; + op->walk([&](Operation *op) { - // TODO: We should have an op interface instead of a hard-coded list of - // interfaces/ops. + // Query BufferViewFlowOpInterface. If the op does not implement that + // interface, try to infer the dependencies from other interfaces that the + // op may implement. + if (auto bufferViewFlowOp = dyn_cast(op)) { + bufferViewFlowOp.populateDependencies(registerDependencies); + for (Value v : op->getResults()) + if (isa(v.getType()) && + bufferViewFlowOp.mayBeTerminalBuffer(v)) + this->terminals.insert(v); + for (Region &r : op->getRegions()) + for (BlockArgument v : r.getArguments()) + if (isa(v.getType()) && + bufferViewFlowOp.mayBeTerminalBuffer(v)) + this->terminals.insert(v); + return WalkResult::advance(); + } // Add additional dependencies created by view changes to the alias list. if (auto viewInterface = dyn_cast(op)) { - dependencies[viewInterface.getViewSource()].insert( - viewInterface->getResult(0)); + registerDependencies(viewInterface.getViewSource(), + viewInterface->getResult(0)); return WalkResult::advance(); } @@ -131,16 +161,30 @@ void BufferViewFlowAnalysis::build(Operation *op) { return WalkResult::advance(); } - // Unknown op: Assume that all operands alias with all results. - for (Value operand : op->getOperands()) { - if (!isa(operand.getType())) - continue; - for (Value result : op->getResults()) { - if (!isa(result.getType())) - continue; - registerDependencies({operand}, {result}); - } + // Region terminators are handled together with RegionBranchOpInterface. + if (isa(op)) + return WalkResult::advance(); + + if (isa(op)) { + // This is an intra-function analysis. We have no information about other + // functions. Conservatively assume that each operand may alias with each + // result. Also mark the results are terminals because the function could + // return newly allocated buffers. + populateTerminalValues(op); + for (Value operand : op->getOperands()) + for (Value result : op->getResults()) + registerDependencies({operand}, {result}); + return WalkResult::advance(); } + + // We have no information about unknown ops. + populateTerminalValues(op); + return WalkResult::advance(); }); } + +bool BufferViewFlowAnalysis::mayBeTerminalBuffer(Value value) const { + assert(isa(value.getType()) && "expected memref"); + return terminals.contains(value); +} diff --git a/mlir/lib/Dialect/MemRef/Transforms/BufferViewFlowOpInterfaceImpl.cpp b/mlir/lib/Dialect/MemRef/Transforms/BufferViewFlowOpInterfaceImpl.cpp new file mode 100644 index 000000000000..bbb269bd0016 --- /dev/null +++ b/mlir/lib/Dialect/MemRef/Transforms/BufferViewFlowOpInterfaceImpl.cpp @@ -0,0 +1,48 @@ +//===- BufferViewFlowOpInterfaceImpl.cpp - Buffer View Flow Analysis ------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "mlir/Dialect/MemRef/Transforms/BufferViewFlowOpInterfaceImpl.h" + +#include "mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.h" +#include "mlir/Dialect/MemRef/IR/MemRef.h" + +using namespace mlir; +using namespace mlir::bufferization; + +namespace mlir { +namespace memref { +namespace { + +struct ReallocOpInterface + : public BufferViewFlowOpInterface::ExternalModel { + void + populateDependencies(Operation *op, + RegisterDependenciesFn registerDependenciesFn) const { + auto reallocOp = cast(op); + // memref.realloc may return the source operand. + registerDependenciesFn(reallocOp.getSource(), reallocOp.getResult()); + } + + bool mayBeTerminalBuffer(Operation *op, Value value) const { + // The return value of memref.realloc is a terminal buffer because the op + // may return a newly allocated buffer. + return true; + } +}; + +} // namespace +} // namespace memref +} // namespace mlir + +void memref::registerBufferViewFlowOpInterfaceExternalModels( + DialectRegistry ®istry) { + registry.addExtension(+[](MLIRContext *ctx, memref::MemRefDialect *dialect) { + ReallocOp::attachInterface(*ctx); + }); +} diff --git a/mlir/lib/Dialect/MemRef/Transforms/CMakeLists.txt b/mlir/lib/Dialect/MemRef/Transforms/CMakeLists.txt index 08b7eab726eb..f150ac7ac2d6 100644 --- a/mlir/lib/Dialect/MemRef/Transforms/CMakeLists.txt +++ b/mlir/lib/Dialect/MemRef/Transforms/CMakeLists.txt @@ -1,5 +1,6 @@ add_mlir_dialect_library(MLIRMemRefTransforms AllocationOpInterfaceImpl.cpp + BufferViewFlowOpInterfaceImpl.cpp ComposeSubView.cpp ExpandOps.cpp ExpandRealloc.cpp @@ -27,6 +28,7 @@ add_mlir_dialect_library(MLIRMemRefTransforms MLIRArithDialect MLIRArithTransforms MLIRBufferizationDialect + MLIRBufferizationTransforms MLIRDialectUtils MLIRFuncDialect MLIRGPUDialect diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel index 5b6e4678a05e..88b46bdb326c 100644 --- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel @@ -10828,6 +10828,36 @@ gentbl_cc_library( ], ) +td_library( + name = "BufferViewFlowOpInterfaceTdFiles", + srcs = [ + "include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.td", + ], + includes = ["include"], + deps = [ + ":OpBaseTdFiles", + ], +) + +gentbl_cc_library( + name = "BufferViewFlowOpInterfaceIncGen", + tbl_outs = [ + ( + ["-gen-op-interface-decls"], + "include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.h.inc", + ), + ( + ["-gen-op-interface-defs"], + "include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.cpp.inc", + ), + ], + tblgen = ":mlir-tblgen", + td_file = "include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.td", + deps = [ + ":BufferViewFlowOpInterfaceTdFiles", + ], +) + td_library( name = "SubsetOpInterfaceTdFiles", srcs = [ @@ -12977,6 +13007,8 @@ cc_library( ":ArithTransforms", ":ArithUtils", ":BufferizationDialect", + ":BufferizationInterfaces", + ":BufferizationTransforms", ":ControlFlowDialect", ":DialectUtils", ":FuncDialect", @@ -13369,6 +13401,7 @@ td_library( includes = ["include"], deps = [ ":AllocationOpInterfaceTdFiles", + ":BufferViewFlowOpInterfaceTdFiles", ":BufferizableOpInterfaceTdFiles", ":CopyOpInterfaceTdFiles", ":DestinationStyleOpInterfaceTdFiles", @@ -13515,11 +13548,13 @@ cc_library( ], hdrs = [ "include/mlir/Dialect/Bufferization/IR/BufferDeallocationOpInterface.h", + "include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.h", "include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h", ], includes = ["include"], deps = [ ":BufferDeallocationOpInterfaceIncGen", + ":BufferViewFlowOpInterfaceIncGen", ":BufferizableOpInterfaceIncGen", ":BufferizationEnumsIncGen", ":IR", @@ -13532,6 +13567,7 @@ cc_library( name = "BufferizationDialect", srcs = [ "lib/Dialect/Bufferization/IR/BufferDeallocationOpInterface.cpp", + "lib/Dialect/Bufferization/IR/BufferViewFlowOpInterface.cpp", "lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp", "lib/Dialect/Bufferization/IR/BufferizationDialect.cpp", "lib/Dialect/Bufferization/IR/BufferizationOps.cpp", @@ -13549,10 +13585,12 @@ cc_library( ":Analysis", ":ArithDialect", ":BufferDeallocationOpInterfaceIncGen", + ":BufferViewFlowOpInterfaceIncGen", ":BufferizableOpInterfaceIncGen", ":BufferizationBaseIncGen", ":BufferizationInterfaces", ":BufferizationOpsIncGen", + ":CallOpInterfaces", ":ControlFlowInterfaces", ":CopyOpInterface", ":DestinationStyleOpInterface", @@ -13602,9 +13640,11 @@ cc_library( ":BufferizationDialect", ":BufferizationInterfaces", ":BufferizationPassIncGen", + ":CallOpInterfaces", ":ControlFlowDialect", ":ControlFlowInterfaces", ":FuncDialect", + ":FunctionInterfaces", ":IR", ":LoopLikeInterface", ":MemRefDialect", -- GitLab From 5d7fd6a04a6748936dece9d90481b2ba4ec97e53 Mon Sep 17 00:00:00 2001 From: yingopq <115543042+yingopq@users.noreply.github.com> Date: Sun, 24 Mar 2024 14:35:42 +0800 Subject: [PATCH 161/306] [Mips] Restore wrong deletion of instruction 'and' in unsigned min/max processing. (#85902) Fix #61881 --- llvm/lib/Target/Mips/MipsExpandPseudo.cpp | 9 ++++ llvm/test/CodeGen/Mips/atomic-min-max.ll | 56 +++++++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/llvm/lib/Target/Mips/MipsExpandPseudo.cpp b/llvm/lib/Target/Mips/MipsExpandPseudo.cpp index bded59439a73..c30129743a96 100644 --- a/llvm/lib/Target/Mips/MipsExpandPseudo.cpp +++ b/llvm/lib/Target/Mips/MipsExpandPseudo.cpp @@ -500,6 +500,15 @@ bool MipsExpandPseudo::expandAtomicBinOpSubword( .addReg(Incr, RegState::Kill) .addImm(ShiftImm); } + } else { + // and OldVal, OldVal, Mask + // and Incr, Incr, Mask + BuildMI(loopMBB, DL, TII->get(Mips::AND), OldVal) + .addReg(OldVal) + .addReg(Mask); + BuildMI(loopMBB, DL, TII->get(Mips::AND), Incr) + .addReg(Incr) + .addReg(Mask); } } // unsigned: sltu Scratch4, oldVal, Incr diff --git a/llvm/test/CodeGen/Mips/atomic-min-max.ll b/llvm/test/CodeGen/Mips/atomic-min-max.ll index bc3643f3947a..a96581bdb39a 100644 --- a/llvm/test/CodeGen/Mips/atomic-min-max.ll +++ b/llvm/test/CodeGen/Mips/atomic-min-max.ll @@ -2146,6 +2146,8 @@ define i16 @test_umax_16(ptr nocapture %ptr, i16 signext %val) { ; MIPS32-NEXT: $BB6_1: # %entry ; MIPS32-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPS32-NEXT: ll $2, 0($6) +; MIPS32-NEXT: and $2, $2, $8 +; MIPS32-NEXT: and $7, $7, $8 ; MIPS32-NEXT: sltu $5, $2, $7 ; MIPS32-NEXT: move $3, $2 ; MIPS32-NEXT: movn $3, $7, $5 @@ -2186,6 +2188,8 @@ define i16 @test_umax_16(ptr nocapture %ptr, i16 signext %val) { ; MIPSEL-NEXT: $BB6_1: # %entry ; MIPSEL-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPSEL-NEXT: ll $2, 0($6) +; MIPSEL-NEXT: and $2, $2, $8 +; MIPSEL-NEXT: and $7, $7, $8 ; MIPSEL-NEXT: sltu $5, $2, $7 ; MIPSEL-NEXT: move $3, $2 ; MIPSEL-NEXT: movn $3, $7, $5 @@ -2225,6 +2229,8 @@ define i16 @test_umax_16(ptr nocapture %ptr, i16 signext %val) { ; MIPSELR6-NEXT: $BB6_1: # %entry ; MIPSELR6-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPSELR6-NEXT: ll $2, 0($6) +; MIPSELR6-NEXT: and $2, $2, $8 +; MIPSELR6-NEXT: and $7, $7, $8 ; MIPSELR6-NEXT: sltu $5, $2, $7 ; MIPSELR6-NEXT: seleqz $3, $2, $5 ; MIPSELR6-NEXT: selnez $5, $7, $5 @@ -2263,6 +2269,8 @@ define i16 @test_umax_16(ptr nocapture %ptr, i16 signext %val) { ; MMEL-NEXT: $BB6_1: # %entry ; MMEL-NEXT: # =>This Inner Loop Header: Depth=1 ; MMEL-NEXT: ll $2, 0($6) +; MMEL-NEXT: and $2, $2, $8 +; MMEL-NEXT: and $7, $7, $8 ; MMEL-NEXT: sltu $5, $2, $7 ; MMEL-NEXT: or $3, $2, $zero ; MMEL-NEXT: movn $3, $7, $5 @@ -2300,6 +2308,8 @@ define i16 @test_umax_16(ptr nocapture %ptr, i16 signext %val) { ; MMELR6-NEXT: $BB6_1: # %entry ; MMELR6-NEXT: # =>This Inner Loop Header: Depth=1 ; MMELR6-NEXT: ll $2, 0($6) +; MMELR6-NEXT: and $2, $2, $8 +; MMELR6-NEXT: and $7, $7, $8 ; MMELR6-NEXT: sltu $5, $2, $7 ; MMELR6-NEXT: seleqz $3, $2, $5 ; MMELR6-NEXT: selnez $5, $7, $5 @@ -2417,6 +2427,8 @@ define i16 @test_umax_16(ptr nocapture %ptr, i16 signext %val) { ; MIPS64EL-NEXT: .LBB6_1: # %entry ; MIPS64EL-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPS64EL-NEXT: ll $2, 0($6) +; MIPS64EL-NEXT: and $2, $2, $8 +; MIPS64EL-NEXT: and $7, $7, $8 ; MIPS64EL-NEXT: sltu $5, $2, $7 ; MIPS64EL-NEXT: move $3, $2 ; MIPS64EL-NEXT: movn $3, $7, $5 @@ -2456,6 +2468,8 @@ define i16 @test_umax_16(ptr nocapture %ptr, i16 signext %val) { ; MIPS64ELR6-NEXT: .LBB6_1: # %entry ; MIPS64ELR6-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPS64ELR6-NEXT: ll $2, 0($6) +; MIPS64ELR6-NEXT: and $2, $2, $8 +; MIPS64ELR6-NEXT: and $7, $7, $8 ; MIPS64ELR6-NEXT: sltu $5, $2, $7 ; MIPS64ELR6-NEXT: seleqz $3, $2, $5 ; MIPS64ELR6-NEXT: selnez $5, $7, $5 @@ -2655,6 +2669,8 @@ define i16 @test_umin_16(ptr nocapture %ptr, i16 signext %val) { ; MIPS32-NEXT: $BB7_1: # %entry ; MIPS32-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPS32-NEXT: ll $2, 0($6) +; MIPS32-NEXT: and $2, $2, $8 +; MIPS32-NEXT: and $7, $7, $8 ; MIPS32-NEXT: sltu $5, $2, $7 ; MIPS32-NEXT: move $3, $2 ; MIPS32-NEXT: movz $3, $7, $5 @@ -2696,6 +2712,8 @@ define i16 @test_umin_16(ptr nocapture %ptr, i16 signext %val) { ; MIPSEL-NEXT: $BB7_1: # %entry ; MIPSEL-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPSEL-NEXT: ll $2, 0($6) +; MIPSEL-NEXT: and $2, $2, $8 +; MIPSEL-NEXT: and $7, $7, $8 ; MIPSEL-NEXT: sltu $5, $2, $7 ; MIPSEL-NEXT: move $3, $2 ; MIPSEL-NEXT: movz $3, $7, $5 @@ -2735,6 +2753,8 @@ define i16 @test_umin_16(ptr nocapture %ptr, i16 signext %val) { ; MIPSELR6-NEXT: $BB7_1: # %entry ; MIPSELR6-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPSELR6-NEXT: ll $2, 0($6) +; MIPSELR6-NEXT: and $2, $2, $8 +; MIPSELR6-NEXT: and $7, $7, $8 ; MIPSELR6-NEXT: sltu $5, $2, $7 ; MIPSELR6-NEXT: selnez $3, $2, $5 ; MIPSELR6-NEXT: seleqz $5, $7, $5 @@ -2773,6 +2793,8 @@ define i16 @test_umin_16(ptr nocapture %ptr, i16 signext %val) { ; MMEL-NEXT: $BB7_1: # %entry ; MMEL-NEXT: # =>This Inner Loop Header: Depth=1 ; MMEL-NEXT: ll $2, 0($6) +; MMEL-NEXT: and $2, $2, $8 +; MMEL-NEXT: and $7, $7, $8 ; MMEL-NEXT: sltu $5, $2, $7 ; MMEL-NEXT: or $3, $2, $zero ; MMEL-NEXT: movz $3, $7, $5 @@ -2810,6 +2832,8 @@ define i16 @test_umin_16(ptr nocapture %ptr, i16 signext %val) { ; MMELR6-NEXT: $BB7_1: # %entry ; MMELR6-NEXT: # =>This Inner Loop Header: Depth=1 ; MMELR6-NEXT: ll $2, 0($6) +; MMELR6-NEXT: and $2, $2, $8 +; MMELR6-NEXT: and $7, $7, $8 ; MMELR6-NEXT: sltu $5, $2, $7 ; MMELR6-NEXT: selnez $3, $2, $5 ; MMELR6-NEXT: seleqz $5, $7, $5 @@ -2927,6 +2951,8 @@ define i16 @test_umin_16(ptr nocapture %ptr, i16 signext %val) { ; MIPS64EL-NEXT: .LBB7_1: # %entry ; MIPS64EL-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPS64EL-NEXT: ll $2, 0($6) +; MIPS64EL-NEXT: and $2, $2, $8 +; MIPS64EL-NEXT: and $7, $7, $8 ; MIPS64EL-NEXT: sltu $5, $2, $7 ; MIPS64EL-NEXT: move $3, $2 ; MIPS64EL-NEXT: movz $3, $7, $5 @@ -2966,6 +2992,8 @@ define i16 @test_umin_16(ptr nocapture %ptr, i16 signext %val) { ; MIPS64ELR6-NEXT: .LBB7_1: # %entry ; MIPS64ELR6-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPS64ELR6-NEXT: ll $2, 0($6) +; MIPS64ELR6-NEXT: and $2, $2, $8 +; MIPS64ELR6-NEXT: and $7, $7, $8 ; MIPS64ELR6-NEXT: sltu $5, $2, $7 ; MIPS64ELR6-NEXT: selnez $3, $2, $5 ; MIPS64ELR6-NEXT: seleqz $5, $7, $5 @@ -4244,6 +4272,8 @@ define i8 @test_umax_8(ptr nocapture %ptr, i8 signext %val) { ; MIPS32-NEXT: $BB10_1: # %entry ; MIPS32-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPS32-NEXT: ll $2, 0($6) +; MIPS32-NEXT: and $2, $2, $8 +; MIPS32-NEXT: and $7, $7, $8 ; MIPS32-NEXT: sltu $5, $2, $7 ; MIPS32-NEXT: move $3, $2 ; MIPS32-NEXT: movn $3, $7, $5 @@ -4284,6 +4314,8 @@ define i8 @test_umax_8(ptr nocapture %ptr, i8 signext %val) { ; MIPSEL-NEXT: $BB10_1: # %entry ; MIPSEL-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPSEL-NEXT: ll $2, 0($6) +; MIPSEL-NEXT: and $2, $2, $8 +; MIPSEL-NEXT: and $7, $7, $8 ; MIPSEL-NEXT: sltu $5, $2, $7 ; MIPSEL-NEXT: move $3, $2 ; MIPSEL-NEXT: movn $3, $7, $5 @@ -4323,6 +4355,8 @@ define i8 @test_umax_8(ptr nocapture %ptr, i8 signext %val) { ; MIPSELR6-NEXT: $BB10_1: # %entry ; MIPSELR6-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPSELR6-NEXT: ll $2, 0($6) +; MIPSELR6-NEXT: and $2, $2, $8 +; MIPSELR6-NEXT: and $7, $7, $8 ; MIPSELR6-NEXT: sltu $5, $2, $7 ; MIPSELR6-NEXT: seleqz $3, $2, $5 ; MIPSELR6-NEXT: selnez $5, $7, $5 @@ -4361,6 +4395,8 @@ define i8 @test_umax_8(ptr nocapture %ptr, i8 signext %val) { ; MMEL-NEXT: $BB10_1: # %entry ; MMEL-NEXT: # =>This Inner Loop Header: Depth=1 ; MMEL-NEXT: ll $2, 0($6) +; MMEL-NEXT: and $2, $2, $8 +; MMEL-NEXT: and $7, $7, $8 ; MMEL-NEXT: sltu $5, $2, $7 ; MMEL-NEXT: or $3, $2, $zero ; MMEL-NEXT: movn $3, $7, $5 @@ -4398,6 +4434,8 @@ define i8 @test_umax_8(ptr nocapture %ptr, i8 signext %val) { ; MMELR6-NEXT: $BB10_1: # %entry ; MMELR6-NEXT: # =>This Inner Loop Header: Depth=1 ; MMELR6-NEXT: ll $2, 0($6) +; MMELR6-NEXT: and $2, $2, $8 +; MMELR6-NEXT: and $7, $7, $8 ; MMELR6-NEXT: sltu $5, $2, $7 ; MMELR6-NEXT: seleqz $3, $2, $5 ; MMELR6-NEXT: selnez $5, $7, $5 @@ -4515,6 +4553,8 @@ define i8 @test_umax_8(ptr nocapture %ptr, i8 signext %val) { ; MIPS64EL-NEXT: .LBB10_1: # %entry ; MIPS64EL-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPS64EL-NEXT: ll $2, 0($6) +; MIPS64EL-NEXT: and $2, $2, $8 +; MIPS64EL-NEXT: and $7, $7, $8 ; MIPS64EL-NEXT: sltu $5, $2, $7 ; MIPS64EL-NEXT: move $3, $2 ; MIPS64EL-NEXT: movn $3, $7, $5 @@ -4554,6 +4594,8 @@ define i8 @test_umax_8(ptr nocapture %ptr, i8 signext %val) { ; MIPS64ELR6-NEXT: .LBB10_1: # %entry ; MIPS64ELR6-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPS64ELR6-NEXT: ll $2, 0($6) +; MIPS64ELR6-NEXT: and $2, $2, $8 +; MIPS64ELR6-NEXT: and $7, $7, $8 ; MIPS64ELR6-NEXT: sltu $5, $2, $7 ; MIPS64ELR6-NEXT: seleqz $3, $2, $5 ; MIPS64ELR6-NEXT: selnez $5, $7, $5 @@ -4753,6 +4795,8 @@ define i8 @test_umin_8(ptr nocapture %ptr, i8 signext %val) { ; MIPS32-NEXT: $BB11_1: # %entry ; MIPS32-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPS32-NEXT: ll $2, 0($6) +; MIPS32-NEXT: and $2, $2, $8 +; MIPS32-NEXT: and $7, $7, $8 ; MIPS32-NEXT: sltu $5, $2, $7 ; MIPS32-NEXT: move $3, $2 ; MIPS32-NEXT: movz $3, $7, $5 @@ -4793,6 +4837,8 @@ define i8 @test_umin_8(ptr nocapture %ptr, i8 signext %val) { ; MIPSEL-NEXT: $BB11_1: # %entry ; MIPSEL-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPSEL-NEXT: ll $2, 0($6) +; MIPSEL-NEXT: and $2, $2, $8 +; MIPSEL-NEXT: and $7, $7, $8 ; MIPSEL-NEXT: sltu $5, $2, $7 ; MIPSEL-NEXT: move $3, $2 ; MIPSEL-NEXT: movz $3, $7, $5 @@ -4832,6 +4878,8 @@ define i8 @test_umin_8(ptr nocapture %ptr, i8 signext %val) { ; MIPSELR6-NEXT: $BB11_1: # %entry ; MIPSELR6-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPSELR6-NEXT: ll $2, 0($6) +; MIPSELR6-NEXT: and $2, $2, $8 +; MIPSELR6-NEXT: and $7, $7, $8 ; MIPSELR6-NEXT: sltu $5, $2, $7 ; MIPSELR6-NEXT: selnez $3, $2, $5 ; MIPSELR6-NEXT: seleqz $5, $7, $5 @@ -4870,6 +4918,8 @@ define i8 @test_umin_8(ptr nocapture %ptr, i8 signext %val) { ; MMEL-NEXT: $BB11_1: # %entry ; MMEL-NEXT: # =>This Inner Loop Header: Depth=1 ; MMEL-NEXT: ll $2, 0($6) +; MMEL-NEXT: and $2, $2, $8 +; MMEL-NEXT: and $7, $7, $8 ; MMEL-NEXT: sltu $5, $2, $7 ; MMEL-NEXT: or $3, $2, $zero ; MMEL-NEXT: movz $3, $7, $5 @@ -4907,6 +4957,8 @@ define i8 @test_umin_8(ptr nocapture %ptr, i8 signext %val) { ; MMELR6-NEXT: $BB11_1: # %entry ; MMELR6-NEXT: # =>This Inner Loop Header: Depth=1 ; MMELR6-NEXT: ll $2, 0($6) +; MMELR6-NEXT: and $2, $2, $8 +; MMELR6-NEXT: and $7, $7, $8 ; MMELR6-NEXT: sltu $5, $2, $7 ; MMELR6-NEXT: selnez $3, $2, $5 ; MMELR6-NEXT: seleqz $5, $7, $5 @@ -5024,6 +5076,8 @@ define i8 @test_umin_8(ptr nocapture %ptr, i8 signext %val) { ; MIPS64EL-NEXT: .LBB11_1: # %entry ; MIPS64EL-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPS64EL-NEXT: ll $2, 0($6) +; MIPS64EL-NEXT: and $2, $2, $8 +; MIPS64EL-NEXT: and $7, $7, $8 ; MIPS64EL-NEXT: sltu $5, $2, $7 ; MIPS64EL-NEXT: move $3, $2 ; MIPS64EL-NEXT: movz $3, $7, $5 @@ -5063,6 +5117,8 @@ define i8 @test_umin_8(ptr nocapture %ptr, i8 signext %val) { ; MIPS64ELR6-NEXT: .LBB11_1: # %entry ; MIPS64ELR6-NEXT: # =>This Inner Loop Header: Depth=1 ; MIPS64ELR6-NEXT: ll $2, 0($6) +; MIPS64ELR6-NEXT: and $2, $2, $8 +; MIPS64ELR6-NEXT: and $7, $7, $8 ; MIPS64ELR6-NEXT: sltu $5, $2, $7 ; MIPS64ELR6-NEXT: selnez $3, $2, $5 ; MIPS64ELR6-NEXT: seleqz $5, $7, $5 -- GitLab From 4acd84e7ccce6a2865f60cd2adc37a335d4f35ce Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Sun, 24 Mar 2024 01:22:48 -0700 Subject: [PATCH 162/306] Revert "[compiler-rt] Also consider SIGPROF as a synchronous signal" (#86416) Reverting #85188 with follow up patches. This reverts commit 362d26366d0175f01ffb6085eb747a6e40f01147. This reverts commit c9bdeabdf4b46fbf1f6a9fcbf9cd61d460b18c08. This reverts commit 6bc6e1ace9fa8453e164fa04b5d9acd5a77e089a. This reverts commit 01fa550ff654d6724e6da54c877032baeddff14b. This reverts commit ddcbab37ac0e5743a8d39be3dd48d967f4c85504. --- compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp | 4 +--- compiler-rt/test/tsan/signal_errno.cpp | 6 +++--- compiler-rt/test/tsan/signal_reset.cpp | 8 ++++---- compiler-rt/test/tsan/signal_sync.cpp | 4 ++-- compiler-rt/test/tsan/signal_thread.cpp | 4 ++-- compiler-rt/test/tsan/signal_thread2.cpp | 4 ++-- 6 files changed, 14 insertions(+), 16 deletions(-) diff --git a/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp b/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp index 810ce69663d0..8ffc703b05ea 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp @@ -126,7 +126,6 @@ const int SIGFPE = 8; const int SIGSEGV = 11; const int SIGPIPE = 13; const int SIGTERM = 15; -const int SIGPROF = 27; #if defined(__mips__) || SANITIZER_FREEBSD || SANITIZER_APPLE || SANITIZER_NETBSD const int SIGBUS = 10; const int SIGSYS = 12; @@ -2180,8 +2179,7 @@ void sighandler(int sig, __sanitizer_siginfo *info, void *ctx) { return; } // Don't mess with synchronous signals. - const bool sync = is_sync_signal(sctx, sig, info) || - (sig == SIGPROF && thr->is_inited && !thr->is_dead); + const bool sync = is_sync_signal(sctx, sig, info); if (sync || // If we are in blocking function, we can safely process it now // (but check if we are in a recursive interceptor, diff --git a/compiler-rt/test/tsan/signal_errno.cpp b/compiler-rt/test/tsan/signal_errno.cpp index 99d4b6d84ca4..7e1fd4b0c5a5 100644 --- a/compiler-rt/test/tsan/signal_errno.cpp +++ b/compiler-rt/test/tsan/signal_errno.cpp @@ -18,7 +18,7 @@ static void MyHandler(int, siginfo_t *s, void *c) { static void* sendsignal(void *p) { barrier_wait(&barrier); - pthread_kill(mainth, SIGALRM); + pthread_kill(mainth, SIGPROF); return 0; } @@ -37,7 +37,7 @@ int main() { mainth = pthread_self(); struct sigaction act = {}; act.sa_sigaction = &MyHandler; - sigaction(SIGALRM, &act, 0); + sigaction(SIGPROF, &act, 0); pthread_t th; pthread_create(&th, 0, sendsignal, 0); loop(); @@ -46,7 +46,7 @@ int main() { } // CHECK: WARNING: ThreadSanitizer: signal handler spoils errno -// CHECK: Signal 14 handler invoked at: +// CHECK: Signal 27 handler invoked at: // CHECK: #0 MyHandler(int, {{(__)?}}siginfo{{(_t)?}}*, void*) {{.*}}signal_errno.cpp // CHECK: main // CHECK: SUMMARY: ThreadSanitizer: signal handler spoils errno{{.*}}MyHandler diff --git a/compiler-rt/test/tsan/signal_reset.cpp b/compiler-rt/test/tsan/signal_reset.cpp index d76b7e5f3b5f..82758d882382 100644 --- a/compiler-rt/test/tsan/signal_reset.cpp +++ b/compiler-rt/test/tsan/signal_reset.cpp @@ -28,12 +28,12 @@ static void* reset(void *p) { struct sigaction act = {}; for (int i = 0; i < 1000000; i++) { act.sa_handler = &handler; - if (sigaction(SIGALRM, &act, 0)) { + if (sigaction(SIGPROF, &act, 0)) { perror("sigaction"); exit(1); } act.sa_handler = SIG_IGN; - if (sigaction(SIGALRM, &act, 0)) { + if (sigaction(SIGPROF, &act, 0)) { perror("sigaction"); exit(1); } @@ -44,7 +44,7 @@ static void* reset(void *p) { int main() { struct sigaction act = {}; act.sa_handler = SIG_IGN; - if (sigaction(SIGALRM, &act, 0)) { + if (sigaction(SIGPROF, &act, 0)) { perror("sigaction"); exit(1); } @@ -53,7 +53,7 @@ int main() { t.it_value.tv_sec = 0; t.it_value.tv_usec = 10; t.it_interval = t.it_value; - if (setitimer(ITIMER_REAL, &t, 0)) { + if (setitimer(ITIMER_PROF, &t, 0)) { perror("setitimer"); exit(1); } diff --git a/compiler-rt/test/tsan/signal_sync.cpp b/compiler-rt/test/tsan/signal_sync.cpp index 878b3f3b88b9..b529a1859f52 100644 --- a/compiler-rt/test/tsan/signal_sync.cpp +++ b/compiler-rt/test/tsan/signal_sync.cpp @@ -30,7 +30,7 @@ int main() { struct sigaction act = {}; act.sa_handler = &handler; - if (sigaction(SIGVTALRM, &act, 0)) { + if (sigaction(SIGPROF, &act, 0)) { perror("sigaction"); exit(1); } @@ -39,7 +39,7 @@ int main() { t.it_value.tv_sec = 0; t.it_value.tv_usec = 10; t.it_interval = t.it_value; - if (setitimer(ITIMER_VIRTUAL, &t, 0)) { + if (setitimer(ITIMER_PROF, &t, 0)) { perror("setitimer"); exit(1); } diff --git a/compiler-rt/test/tsan/signal_thread.cpp b/compiler-rt/test/tsan/signal_thread.cpp index 7bba8159bf38..aa91d1ddeb10 100644 --- a/compiler-rt/test/tsan/signal_thread.cpp +++ b/compiler-rt/test/tsan/signal_thread.cpp @@ -24,7 +24,7 @@ static void* thr(void *p) { int main() { struct sigaction act = {}; act.sa_handler = &handler; - if (sigaction(SIGVTALRM, &act, 0)) { + if (sigaction(SIGPROF, &act, 0)) { perror("sigaction"); exit(1); } @@ -33,7 +33,7 @@ int main() { t.it_value.tv_sec = 0; t.it_value.tv_usec = 10; t.it_interval = t.it_value; - if (setitimer(ITIMER_VIRTUAL, &t, 0)) { + if (setitimer(ITIMER_PROF, &t, 0)) { perror("setitimer"); exit(1); } diff --git a/compiler-rt/test/tsan/signal_thread2.cpp b/compiler-rt/test/tsan/signal_thread2.cpp index 5236628e13b6..9bde4f70b39d 100644 --- a/compiler-rt/test/tsan/signal_thread2.cpp +++ b/compiler-rt/test/tsan/signal_thread2.cpp @@ -40,7 +40,7 @@ static void *thr(void *p) { int main() { struct sigaction act = {}; act.sa_handler = &handler; - if (sigaction(SIGALRM, &act, 0)) { + if (sigaction(SIGPROF, &act, 0)) { perror("sigaction"); exit(1); } @@ -49,7 +49,7 @@ int main() { t.it_value.tv_sec = 0; t.it_value.tv_usec = 10; t.it_interval = t.it_value; - if (setitimer(ITIMER_REAL, &t, 0)) { + if (setitimer(ITIMER_PROF, &t, 0)) { perror("setitimer"); exit(1); } -- GitLab From 4d69855e9d380ecb7c1f7a64c7b37258fe36f525 Mon Sep 17 00:00:00 2001 From: AtariDreams <83477269+AtariDreams@users.noreply.github.com> Date: Sun, 24 Mar 2024 06:48:04 -0400 Subject: [PATCH 163/306] [flang] Silence MSVC warning about shifts (NFC) (#83737) Yes, 64-bit shifts are intended. --- flang/lib/Optimizer/Builder/IntrinsicCall.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp index eb8f5135ff12..ea1ef1f08aba 100644 --- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp +++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp @@ -3883,7 +3883,7 @@ mlir::Value IntrinsicLibrary::genIeeeClass(mlir::Type resultType, int pos = 3 + highSignificandSize; mlir::Value index = builder.create( loc, builder.create(loc, intVal, signShift), - createIntegerConstant(1 << pos)); + createIntegerConstant(1ULL << pos)); // [e] exponent != 0 mlir::Value exponent = @@ -3895,7 +3895,7 @@ mlir::Value IntrinsicLibrary::genIeeeClass(mlir::Type resultType, loc, builder.create( loc, mlir::arith::CmpIPredicate::ne, exponent, zero), - createIntegerConstant(1 << --pos), zero)); + createIntegerConstant(1ULL << --pos), zero)); // [m] exponent == 1..1 (max exponent) index = builder.create( @@ -3904,7 +3904,7 @@ mlir::Value IntrinsicLibrary::genIeeeClass(mlir::Type resultType, loc, builder.create( loc, mlir::arith::CmpIPredicate::eq, exponent, exponentMask), - createIntegerConstant(1 << --pos), zero)); + createIntegerConstant(1ULL << --pos), zero)); // [l] low-order significand != 0 index = builder.create( @@ -3916,7 +3916,7 @@ mlir::Value IntrinsicLibrary::genIeeeClass(mlir::Type resultType, builder.create(loc, intVal, lowSignificandMask), zero), - createIntegerConstant(1 << --pos), zero)); + createIntegerConstant(1ULL << --pos), zero)); // [h] high-order significand (1 or 2 bits) index = builder.create( -- GitLab From 6c6fe4b2aea8631001b11abee62146d4aca01cee Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Sun, 24 Mar 2024 11:33:51 +0000 Subject: [PATCH 164/306] [X86] known-never-zero.ll - add 32-bit test coverage Enabled vector coverage as well: i686+SSE2 and x64_64+AVX Should improve test quality for #85722 --- llvm/test/CodeGen/X86/known-never-zero.ll | 1708 ++++++++++++++------- 1 file changed, 1161 insertions(+), 547 deletions(-) diff --git a/llvm/test/CodeGen/X86/known-never-zero.ll b/llvm/test/CodeGen/X86/known-never-zero.ll index cc9862769f2b..423516bc3271 100644 --- a/llvm/test/CodeGen/X86/known-never-zero.ll +++ b/llvm/test/CodeGen/X86/known-never-zero.ll @@ -1,5 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefix=CHECK +; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=X86 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=X64 ;; Use cttz to test if we properly prove never-zero. There is a very ;; simple transform from cttz -> cttz_zero_undef if its operand is @@ -18,41 +19,70 @@ declare i32 @llvm.fshl.i32(i32, i32, i32) declare i32 @llvm.fshr.i32(i32, i32, i32) define i32 @or_known_nonzero(i32 %x) { -; CHECK-LABEL: or_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: orl $1, %edi -; CHECK-NEXT: rep bsfl %edi, %eax -; CHECK-NEXT: retq +; X86-LABEL: or_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: orl $1, %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: or_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: orl $1, %edi +; X64-NEXT: rep bsfl %edi, %eax +; X64-NEXT: retq %z = or i32 %x, 1 %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @or_maybe_zero(i32 %x, i32 %y) { -; CHECK-LABEL: or_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: orl %esi, %edi -; CHECK-NEXT: je .LBB1_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %edi, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB1_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: or_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: orl {{[0-9]+}}(%esp), %eax +; X86-NEXT: je .LBB1_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB1_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: or_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: orl %esi, %edi +; X64-NEXT: je .LBB1_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %edi, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB1_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = or i32 %x, %y %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @select_known_nonzero(i1 %c, i32 %x) { -; CHECK-LABEL: select_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: orl $1, %esi -; CHECK-NEXT: testb $1, %dil -; CHECK-NEXT: movl $122, %eax -; CHECK-NEXT: cmovnel %esi, %eax -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq +; X86-LABEL: select_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: orl $1, %eax +; X86-NEXT: testb $1, {{[0-9]+}}(%esp) +; X86-NEXT: movl $122, %ecx +; X86-NEXT: cmovnel %eax, %ecx +; X86-NEXT: rep bsfl %ecx, %eax +; X86-NEXT: retl +; +; X64-LABEL: select_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: orl $1, %esi +; X64-NEXT: testb $1, %dil +; X64-NEXT: movl $122, %eax +; X64-NEXT: cmovnel %esi, %eax +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq %y = or i32 %x, 1 %z = select i1 %c, i32 %y, i32 122 %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -60,20 +90,36 @@ define i32 @select_known_nonzero(i1 %c, i32 %x) { } define i32 @select_maybe_zero(i1 %c, i32 %x) { -; CHECK-LABEL: select_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: orl $1, %esi -; CHECK-NEXT: xorl %eax, %eax -; CHECK-NEXT: testb $1, %dil -; CHECK-NEXT: cmovnel %esi, %eax -; CHECK-NEXT: testl %eax, %eax -; CHECK-NEXT: je .LBB3_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB3_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: select_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: orl $1, %ecx +; X86-NEXT: xorl %eax, %eax +; X86-NEXT: testb $1, {{[0-9]+}}(%esp) +; X86-NEXT: cmovnel %ecx, %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB3_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB3_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: select_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: orl $1, %esi +; X64-NEXT: xorl %eax, %eax +; X64-NEXT: testb $1, %dil +; X64-NEXT: cmovnel %esi, %eax +; X64-NEXT: testl %eax, %eax +; X64-NEXT: je .LBB3_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB3_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %y = or i32 %x, 1 %z = select i1 %c, i32 %y, i32 0 %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -81,28 +127,45 @@ define i32 @select_maybe_zero(i1 %c, i32 %x) { } define i32 @shl_known_nonzero_1s_bit_set(i32 %x) { -; CHECK-LABEL: shl_known_nonzero_1s_bit_set: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %ecx -; CHECK-NEXT: movl $123, %eax -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: shll %cl, %eax -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq +; X86-LABEL: shl_known_nonzero_1s_bit_set: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $123, %eax +; X86-NEXT: shll %cl, %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: shl_known_nonzero_1s_bit_set: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %ecx +; X64-NEXT: movl $123, %eax +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: shll %cl, %eax +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq %z = shl i32 123, %x %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @shl_known_nonzero_nsw(i32 %x, i32 %yy) { -; CHECK-LABEL: shl_known_nonzero_nsw: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %ecx -; CHECK-NEXT: orl $256, %esi # imm = 0x100 -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: shll %cl, %esi -; CHECK-NEXT: rep bsfl %esi, %eax -; CHECK-NEXT: retq +; X86-LABEL: shl_known_nonzero_nsw: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $256, %eax # imm = 0x100 +; X86-NEXT: orl {{[0-9]+}}(%esp), %eax +; X86-NEXT: shll %cl, %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: shl_known_nonzero_nsw: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %ecx +; X64-NEXT: orl $256, %esi # imm = 0x100 +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: shll %cl, %esi +; X64-NEXT: rep bsfl %esi, %eax +; X64-NEXT: retq %y = or i32 %yy, 256 %z = shl nsw i32 %y, %x %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -110,14 +173,23 @@ define i32 @shl_known_nonzero_nsw(i32 %x, i32 %yy) { } define i32 @shl_known_nonzero_nuw(i32 %x, i32 %yy) { -; CHECK-LABEL: shl_known_nonzero_nuw: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %ecx -; CHECK-NEXT: orl $256, %esi # imm = 0x100 -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: shll %cl, %esi -; CHECK-NEXT: rep bsfl %esi, %eax -; CHECK-NEXT: retq +; X86-LABEL: shl_known_nonzero_nuw: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $256, %eax # imm = 0x100 +; X86-NEXT: orl {{[0-9]+}}(%esp), %eax +; X86-NEXT: shll %cl, %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: shl_known_nonzero_nuw: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %ecx +; X64-NEXT: orl $256, %esi # imm = 0x100 +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: shll %cl, %esi +; X64-NEXT: rep bsfl %esi, %eax +; X64-NEXT: retq %y = or i32 %yy, 256 %z = shl nuw i32 %y, %x %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -125,67 +197,116 @@ define i32 @shl_known_nonzero_nuw(i32 %x, i32 %yy) { } define i32 @shl_maybe_zero(i32 %x, i32 %y) { -; CHECK-LABEL: shl_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %ecx -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: shll %cl, %esi -; CHECK-NEXT: testl %esi, %esi -; CHECK-NEXT: je .LBB7_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %esi, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB7_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: shl_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: shll %cl, %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB7_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB7_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: shl_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %ecx +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: shll %cl, %esi +; X64-NEXT: testl %esi, %esi +; X64-NEXT: je .LBB7_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %esi, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB7_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = shl nuw nsw i32 %y, %x %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @uaddsat_known_nonzero(i32 %x) { -; CHECK-LABEL: uaddsat_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: incl %edi -; CHECK-NEXT: movl $-1, %eax -; CHECK-NEXT: cmovnel %edi, %eax -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq +; X86-LABEL: uaddsat_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: incl %eax +; X86-NEXT: movl $-1, %ecx +; X86-NEXT: cmovnel %eax, %ecx +; X86-NEXT: rep bsfl %ecx, %eax +; X86-NEXT: retl +; +; X64-LABEL: uaddsat_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: incl %edi +; X64-NEXT: movl $-1, %eax +; X64-NEXT: cmovnel %edi, %eax +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq %z = call i32 @llvm.uadd.sat.i32(i32 %x, i32 1) %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @uaddsat_maybe_zero(i32 %x, i32 %y) { -; CHECK-LABEL: uaddsat_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: addl %esi, %edi -; CHECK-NEXT: movl $-1, %eax -; CHECK-NEXT: cmovael %edi, %eax -; CHECK-NEXT: testl %eax, %eax -; CHECK-NEXT: je .LBB9_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB9_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: uaddsat_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: addl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $-1, %eax +; X86-NEXT: cmovael %ecx, %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB9_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB9_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: uaddsat_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: addl %esi, %edi +; X64-NEXT: movl $-1, %eax +; X64-NEXT: cmovael %edi, %eax +; X64-NEXT: testl %eax, %eax +; X64-NEXT: je .LBB9_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB9_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = call i32 @llvm.uadd.sat.i32(i32 %x, i32 %y) %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @umax_known_nonzero(i32 %x, i32 %y) { -; CHECK-LABEL: umax_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %esi, %ecx -; CHECK-NEXT: movl $4, %eax -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: shll %cl, %eax -; CHECK-NEXT: cmpl %eax, %edi -; CHECK-NEXT: cmoval %edi, %eax -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq +; X86-LABEL: umax_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $4, %edx +; X86-NEXT: shll %cl, %edx +; X86-NEXT: cmpl %edx, %eax +; X86-NEXT: cmoval %eax, %edx +; X86-NEXT: rep bsfl %edx, %eax +; X86-NEXT: retl +; +; X64-LABEL: umax_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: movl %esi, %ecx +; X64-NEXT: movl $4, %eax +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: shll %cl, %eax +; X64-NEXT: cmpl %eax, %edi +; X64-NEXT: cmoval %edi, %eax +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq %yy = shl nuw i32 4, %y %z = call i32 @llvm.umax.i32(i32 %x, i32 %yy) %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -193,35 +314,62 @@ define i32 @umax_known_nonzero(i32 %x, i32 %y) { } define i32 @umax_maybe_zero(i32 %x, i32 %y) { -; CHECK-LABEL: umax_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: cmpl %esi, %edi -; CHECK-NEXT: cmoval %edi, %esi -; CHECK-NEXT: testl %esi, %esi -; CHECK-NEXT: je .LBB11_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %esi, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB11_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: umax_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: cmpl %eax, %ecx +; X86-NEXT: cmoval %ecx, %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB11_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB11_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: umax_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: cmpl %esi, %edi +; X64-NEXT: cmoval %edi, %esi +; X64-NEXT: testl %esi, %esi +; X64-NEXT: je .LBB11_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %esi, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB11_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = call i32 @llvm.umax.i32(i32 %x, i32 %y) %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @umin_known_nonzero(i32 %xx, i32 %yy) { -; CHECK-LABEL: umin_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %ecx -; CHECK-NEXT: movl $4, %eax -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: shll %cl, %eax -; CHECK-NEXT: addl $4, %esi -; CHECK-NEXT: cmpl %esi, %eax -; CHECK-NEXT: cmovbl %eax, %esi -; CHECK-NEXT: rep bsfl %esi, %eax -; CHECK-NEXT: retq +; X86-LABEL: umin_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $4, %edx +; X86-NEXT: shll %cl, %edx +; X86-NEXT: addl $4, %eax +; X86-NEXT: cmpl %eax, %edx +; X86-NEXT: cmovbl %edx, %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: umin_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %ecx +; X64-NEXT: movl $4, %eax +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: shll %cl, %eax +; X64-NEXT: addl $4, %esi +; X64-NEXT: cmpl %esi, %eax +; X64-NEXT: cmovbl %eax, %esi +; X64-NEXT: rep bsfl %esi, %eax +; X64-NEXT: retq %x = shl nuw i32 4, %xx %y = add nuw nsw i32 %yy, 4 %z = call i32 @llvm.umin.i32(i32 %x, i32 %y) @@ -230,36 +378,63 @@ define i32 @umin_known_nonzero(i32 %xx, i32 %yy) { } define i32 @umin_maybe_zero(i32 %x, i32 %y) { -; CHECK-LABEL: umin_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: cmpl $54, %edi -; CHECK-NEXT: movl $54, %eax -; CHECK-NEXT: cmovbl %edi, %eax -; CHECK-NEXT: testl %eax, %eax -; CHECK-NEXT: je .LBB13_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB13_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: umin_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: cmpl $54, %ecx +; X86-NEXT: movl $54, %eax +; X86-NEXT: cmovbl %ecx, %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB13_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB13_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: umin_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: cmpl $54, %edi +; X64-NEXT: movl $54, %eax +; X64-NEXT: cmovbl %edi, %eax +; X64-NEXT: testl %eax, %eax +; X64-NEXT: je .LBB13_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB13_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = call i32 @llvm.umin.i32(i32 %x, i32 54) %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @smin_known_nonzero(i32 %xx, i32 %yy) { -; CHECK-LABEL: smin_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %ecx -; CHECK-NEXT: movl $4, %eax -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: shll %cl, %eax -; CHECK-NEXT: addl $4, %esi -; CHECK-NEXT: cmpl %esi, %eax -; CHECK-NEXT: cmovll %eax, %esi -; CHECK-NEXT: rep bsfl %esi, %eax -; CHECK-NEXT: retq +; X86-LABEL: smin_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $4, %edx +; X86-NEXT: shll %cl, %edx +; X86-NEXT: addl $4, %eax +; X86-NEXT: cmpl %eax, %edx +; X86-NEXT: cmovll %edx, %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: smin_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %ecx +; X64-NEXT: movl $4, %eax +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: shll %cl, %eax +; X64-NEXT: addl $4, %esi +; X64-NEXT: cmpl %esi, %eax +; X64-NEXT: cmovll %eax, %esi +; X64-NEXT: rep bsfl %esi, %eax +; X64-NEXT: retq %x = shl nuw i32 4, %xx %y = add nuw nsw i32 %yy, 4 %z = call i32 @llvm.smin.i32(i32 %x, i32 %y) @@ -268,36 +443,63 @@ define i32 @smin_known_nonzero(i32 %xx, i32 %yy) { } define i32 @smin_maybe_zero(i32 %x, i32 %y) { -; CHECK-LABEL: smin_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: cmpl $54, %edi -; CHECK-NEXT: movl $54, %eax -; CHECK-NEXT: cmovll %edi, %eax -; CHECK-NEXT: testl %eax, %eax -; CHECK-NEXT: je .LBB15_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB15_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: smin_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: cmpl $54, %ecx +; X86-NEXT: movl $54, %eax +; X86-NEXT: cmovll %ecx, %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB15_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB15_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: smin_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: cmpl $54, %edi +; X64-NEXT: movl $54, %eax +; X64-NEXT: cmovll %edi, %eax +; X64-NEXT: testl %eax, %eax +; X64-NEXT: je .LBB15_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB15_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = call i32 @llvm.smin.i32(i32 %x, i32 54) %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @smax_known_nonzero(i32 %xx, i32 %yy) { -; CHECK-LABEL: smax_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %ecx -; CHECK-NEXT: movl $4, %eax -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: shll %cl, %eax -; CHECK-NEXT: addl $4, %esi -; CHECK-NEXT: cmpl %esi, %eax -; CHECK-NEXT: cmovgl %eax, %esi -; CHECK-NEXT: rep bsfl %esi, %eax -; CHECK-NEXT: retq +; X86-LABEL: smax_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $4, %edx +; X86-NEXT: shll %cl, %edx +; X86-NEXT: addl $4, %eax +; X86-NEXT: cmpl %eax, %edx +; X86-NEXT: cmovgl %edx, %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: smax_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %ecx +; X64-NEXT: movl $4, %eax +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: shll %cl, %eax +; X64-NEXT: addl $4, %esi +; X64-NEXT: cmpl %esi, %eax +; X64-NEXT: cmovgl %eax, %esi +; X64-NEXT: rep bsfl %esi, %eax +; X64-NEXT: retq %x = shl nuw i32 4, %xx %y = add nuw nsw i32 %yy, 4 %z = call i32 @llvm.smax.i32(i32 %x, i32 %y) @@ -306,35 +508,61 @@ define i32 @smax_known_nonzero(i32 %xx, i32 %yy) { } define i32 @smax_maybe_zero(i32 %x, i32 %y) { -; CHECK-LABEL: smax_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: cmpl $55, %edi -; CHECK-NEXT: movl $54, %eax -; CHECK-NEXT: cmovgel %edi, %eax -; CHECK-NEXT: bsfl %eax, %ecx -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: cmovnel %ecx, %eax -; CHECK-NEXT: retq +; X86-LABEL: smax_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: cmpl $55, %eax +; X86-NEXT: movl $54, %ecx +; X86-NEXT: cmovgel %eax, %ecx +; X86-NEXT: bsfl %ecx, %ecx +; X86-NEXT: movl $32, %eax +; X86-NEXT: cmovnel %ecx, %eax +; X86-NEXT: retl +; +; X64-LABEL: smax_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: cmpl $55, %edi +; X64-NEXT: movl $54, %eax +; X64-NEXT: cmovgel %edi, %eax +; X64-NEXT: bsfl %eax, %ecx +; X64-NEXT: movl $32, %eax +; X64-NEXT: cmovnel %ecx, %eax +; X64-NEXT: retq %z = call i32 @llvm.smax.i32(i32 %x, i32 54) %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @rotr_known_nonzero(i32 %xx, i32 %y) { -; CHECK-LABEL: rotr_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %esi, %ecx -; CHECK-NEXT: orl $256, %edi # imm = 0x100 -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: rorl %cl, %edi -; CHECK-NEXT: testl %edi, %edi -; CHECK-NEXT: je .LBB18_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %edi, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB18_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: rotr_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $256, %eax # imm = 0x100 +; X86-NEXT: orl {{[0-9]+}}(%esp), %eax +; X86-NEXT: rorl %cl, %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB18_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB18_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: rotr_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: movl %esi, %ecx +; X64-NEXT: orl $256, %edi # imm = 0x100 +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: rorl %cl, %edi +; X64-NEXT: testl %edi, %edi +; X64-NEXT: je .LBB18_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %edi, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB18_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %x = or i32 %xx, 256 %shr = lshr i32 %x, %y %sub = sub i32 32, %y @@ -345,19 +573,33 @@ define i32 @rotr_known_nonzero(i32 %xx, i32 %y) { } define i32 @rotr_maybe_zero(i32 %x, i32 %y) { -; CHECK-LABEL: rotr_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %esi, %ecx -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: rorl %cl, %edi -; CHECK-NEXT: testl %edi, %edi -; CHECK-NEXT: je .LBB19_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %edi, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB19_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: rotr_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: rorl %cl, %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB19_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB19_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: rotr_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: movl %esi, %ecx +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: rorl %cl, %edi +; X64-NEXT: testl %edi, %edi +; X64-NEXT: je .LBB19_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %edi, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB19_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %shr = lshr i32 %x, %y %sub = sub i32 32, %y %shl = shl i32 %x, %sub @@ -367,14 +609,23 @@ define i32 @rotr_maybe_zero(i32 %x, i32 %y) { } define i32 @rotr_with_fshr_known_nonzero(i32 %xx, i32 %y) { -; CHECK-LABEL: rotr_with_fshr_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %esi, %ecx -; CHECK-NEXT: orl $256, %edi # imm = 0x100 -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: rorl %cl, %edi -; CHECK-NEXT: rep bsfl %edi, %eax -; CHECK-NEXT: retq +; X86-LABEL: rotr_with_fshr_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $256, %eax # imm = 0x100 +; X86-NEXT: orl {{[0-9]+}}(%esp), %eax +; X86-NEXT: rorl %cl, %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: rotr_with_fshr_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: movl %esi, %ecx +; X64-NEXT: orl $256, %edi # imm = 0x100 +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: rorl %cl, %edi +; X64-NEXT: rep bsfl %edi, %eax +; X64-NEXT: retq %x = or i32 %xx, 256 %z = call i32 @llvm.fshr.i32(i32 %x, i32 %x, i32 %y) %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -382,39 +633,68 @@ define i32 @rotr_with_fshr_known_nonzero(i32 %xx, i32 %y) { } define i32 @rotr_with_fshr_maybe_zero(i32 %x, i32 %y) { -; CHECK-LABEL: rotr_with_fshr_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %esi, %ecx -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: rorl %cl, %edi -; CHECK-NEXT: testl %edi, %edi -; CHECK-NEXT: je .LBB21_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %edi, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB21_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: rotr_with_fshr_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: rorl %cl, %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB21_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB21_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: rotr_with_fshr_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: movl %esi, %ecx +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: rorl %cl, %edi +; X64-NEXT: testl %edi, %edi +; X64-NEXT: je .LBB21_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %edi, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB21_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = call i32 @llvm.fshr.i32(i32 %x, i32 %x, i32 %y) %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @rotl_known_nonzero(i32 %xx, i32 %y) { -; CHECK-LABEL: rotl_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %esi, %ecx -; CHECK-NEXT: orl $256, %edi # imm = 0x100 -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: roll %cl, %edi -; CHECK-NEXT: testl %edi, %edi -; CHECK-NEXT: je .LBB22_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %edi, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB22_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: rotl_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $256, %eax # imm = 0x100 +; X86-NEXT: orl {{[0-9]+}}(%esp), %eax +; X86-NEXT: roll %cl, %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB22_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB22_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: rotl_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: movl %esi, %ecx +; X64-NEXT: orl $256, %edi # imm = 0x100 +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: roll %cl, %edi +; X64-NEXT: testl %edi, %edi +; X64-NEXT: je .LBB22_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %edi, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB22_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %x = or i32 %xx, 256 %shl = shl i32 %x, %y %sub = sub i32 32, %y @@ -425,19 +705,33 @@ define i32 @rotl_known_nonzero(i32 %xx, i32 %y) { } define i32 @rotl_maybe_zero(i32 %x, i32 %y) { -; CHECK-LABEL: rotl_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %esi, %ecx -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: roll %cl, %edi -; CHECK-NEXT: testl %edi, %edi -; CHECK-NEXT: je .LBB23_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %edi, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB23_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: rotl_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: roll %cl, %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB23_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB23_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: rotl_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: movl %esi, %ecx +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: roll %cl, %edi +; X64-NEXT: testl %edi, %edi +; X64-NEXT: je .LBB23_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %edi, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB23_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %shl = shl i32 %x, %y %sub = sub i32 32, %y %shr = lshr i32 %x, %sub @@ -447,14 +741,23 @@ define i32 @rotl_maybe_zero(i32 %x, i32 %y) { } define i32 @rotl_with_fshl_known_nonzero(i32 %xx, i32 %y) { -; CHECK-LABEL: rotl_with_fshl_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %esi, %ecx -; CHECK-NEXT: orl $256, %edi # imm = 0x100 -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: roll %cl, %edi -; CHECK-NEXT: rep bsfl %edi, %eax -; CHECK-NEXT: retq +; X86-LABEL: rotl_with_fshl_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $256, %eax # imm = 0x100 +; X86-NEXT: orl {{[0-9]+}}(%esp), %eax +; X86-NEXT: roll %cl, %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: rotl_with_fshl_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: movl %esi, %ecx +; X64-NEXT: orl $256, %edi # imm = 0x100 +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: roll %cl, %edi +; X64-NEXT: rep bsfl %edi, %eax +; X64-NEXT: retq %x = or i32 %xx, 256 %z = call i32 @llvm.fshl.i32(i32 %x, i32 %x, i32 %y) %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -462,47 +765,78 @@ define i32 @rotl_with_fshl_known_nonzero(i32 %xx, i32 %y) { } define i32 @rotl_with_fshl_maybe_zero(i32 %x, i32 %y) { -; CHECK-LABEL: rotl_with_fshl_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %esi, %ecx -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: roll %cl, %edi -; CHECK-NEXT: testl %edi, %edi -; CHECK-NEXT: je .LBB25_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %edi, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB25_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: rotl_with_fshl_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: roll %cl, %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB25_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB25_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: rotl_with_fshl_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: movl %esi, %ecx +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: roll %cl, %edi +; X64-NEXT: testl %edi, %edi +; X64-NEXT: je .LBB25_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %edi, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB25_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = call i32 @llvm.fshl.i32(i32 %x, i32 %x, i32 %y) %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @sra_known_nonzero_sign_bit_set(i32 %x) { -; CHECK-LABEL: sra_known_nonzero_sign_bit_set: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %ecx -; CHECK-NEXT: movl $-2147360405, %eax # imm = 0x8001E16B -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: sarl %cl, %eax -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq +; X86-LABEL: sra_known_nonzero_sign_bit_set: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $-2147360405, %eax # imm = 0x8001E16B +; X86-NEXT: sarl %cl, %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: sra_known_nonzero_sign_bit_set: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %ecx +; X64-NEXT: movl $-2147360405, %eax # imm = 0x8001E16B +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: sarl %cl, %eax +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq %z = ashr i32 2147606891, %x %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @sra_known_nonzero_exact(i32 %x, i32 %yy) { -; CHECK-LABEL: sra_known_nonzero_exact: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %ecx -; CHECK-NEXT: orl $256, %esi # imm = 0x100 -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: sarl %cl, %esi -; CHECK-NEXT: rep bsfl %esi, %eax -; CHECK-NEXT: retq +; X86-LABEL: sra_known_nonzero_exact: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $256, %eax # imm = 0x100 +; X86-NEXT: orl {{[0-9]+}}(%esp), %eax +; X86-NEXT: sarl %cl, %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: sra_known_nonzero_exact: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %ecx +; X64-NEXT: orl $256, %esi # imm = 0x100 +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: sarl %cl, %esi +; X64-NEXT: rep bsfl %esi, %eax +; X64-NEXT: retq %y = or i32 %yy, 256 %z = ashr exact i32 %y, %x %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -510,47 +844,78 @@ define i32 @sra_known_nonzero_exact(i32 %x, i32 %yy) { } define i32 @sra_maybe_zero(i32 %x, i32 %y) { -; CHECK-LABEL: sra_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %ecx -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: sarl %cl, %esi -; CHECK-NEXT: testl %esi, %esi -; CHECK-NEXT: je .LBB28_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %esi, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB28_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: sra_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: sarl %cl, %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB28_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB28_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: sra_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %ecx +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: sarl %cl, %esi +; X64-NEXT: testl %esi, %esi +; X64-NEXT: je .LBB28_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %esi, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB28_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = ashr exact i32 %y, %x %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @srl_known_nonzero_sign_bit_set(i32 %x) { -; CHECK-LABEL: srl_known_nonzero_sign_bit_set: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %ecx -; CHECK-NEXT: movl $-2147360405, %eax # imm = 0x8001E16B -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: shrl %cl, %eax -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq +; X86-LABEL: srl_known_nonzero_sign_bit_set: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $-2147360405, %eax # imm = 0x8001E16B +; X86-NEXT: shrl %cl, %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: srl_known_nonzero_sign_bit_set: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %ecx +; X64-NEXT: movl $-2147360405, %eax # imm = 0x8001E16B +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: shrl %cl, %eax +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq %z = lshr i32 2147606891, %x %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @srl_known_nonzero_exact(i32 %x, i32 %yy) { -; CHECK-LABEL: srl_known_nonzero_exact: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %ecx -; CHECK-NEXT: orl $256, %esi # imm = 0x100 -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: shrl %cl, %esi -; CHECK-NEXT: rep bsfl %esi, %eax -; CHECK-NEXT: retq +; X86-LABEL: srl_known_nonzero_exact: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $256, %eax # imm = 0x100 +; X86-NEXT: orl {{[0-9]+}}(%esp), %eax +; X86-NEXT: shrl %cl, %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: srl_known_nonzero_exact: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %ecx +; X64-NEXT: orl $256, %esi # imm = 0x100 +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: shrl %cl, %esi +; X64-NEXT: rep bsfl %esi, %eax +; X64-NEXT: retq %y = or i32 %yy, 256 %z = lshr exact i32 %y, %x %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -558,33 +923,56 @@ define i32 @srl_known_nonzero_exact(i32 %x, i32 %yy) { } define i32 @srl_maybe_zero(i32 %x, i32 %y) { -; CHECK-LABEL: srl_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %ecx -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: shrl %cl, %esi -; CHECK-NEXT: testl %esi, %esi -; CHECK-NEXT: je .LBB31_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %esi, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB31_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: srl_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: shrl %cl, %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB31_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB31_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: srl_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %ecx +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: shrl %cl, %esi +; X64-NEXT: testl %esi, %esi +; X64-NEXT: je .LBB31_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %esi, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB31_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = lshr exact i32 %y, %x %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @udiv_known_nonzero(i32 %xx, i32 %y) { -; CHECK-LABEL: udiv_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %eax -; CHECK-NEXT: orl $64, %eax -; CHECK-NEXT: xorl %edx, %edx -; CHECK-NEXT: divl %esi -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq +; X86-LABEL: udiv_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: orl $64, %eax +; X86-NEXT: xorl %edx, %edx +; X86-NEXT: divl {{[0-9]+}}(%esp) +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: udiv_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %eax +; X64-NEXT: orl $64, %eax +; X64-NEXT: xorl %edx, %edx +; X64-NEXT: divl %esi +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq %x = or i32 %xx, 64 %z = udiv exact i32 %x, %y %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -592,33 +980,56 @@ define i32 @udiv_known_nonzero(i32 %xx, i32 %y) { } define i32 @udiv_maybe_zero(i32 %x, i32 %y) { -; CHECK-LABEL: udiv_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %eax -; CHECK-NEXT: xorl %edx, %edx -; CHECK-NEXT: divl %esi -; CHECK-NEXT: testl %eax, %eax -; CHECK-NEXT: je .LBB33_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB33_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: udiv_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: xorl %edx, %edx +; X86-NEXT: divl {{[0-9]+}}(%esp) +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB33_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB33_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: udiv_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %eax +; X64-NEXT: xorl %edx, %edx +; X64-NEXT: divl %esi +; X64-NEXT: testl %eax, %eax +; X64-NEXT: je .LBB33_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB33_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = udiv exact i32 %x, %y %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @sdiv_known_nonzero(i32 %xx, i32 %y) { -; CHECK-LABEL: sdiv_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %eax -; CHECK-NEXT: orl $64, %eax -; CHECK-NEXT: cltd -; CHECK-NEXT: idivl %esi -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq +; X86-LABEL: sdiv_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: orl $64, %eax +; X86-NEXT: cltd +; X86-NEXT: idivl {{[0-9]+}}(%esp) +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: sdiv_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %eax +; X64-NEXT: orl $64, %eax +; X64-NEXT: cltd +; X64-NEXT: idivl %esi +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq %x = or i32 %xx, 64 %z = sdiv exact i32 %x, %y %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -626,31 +1037,53 @@ define i32 @sdiv_known_nonzero(i32 %xx, i32 %y) { } define i32 @sdiv_maybe_zero(i32 %x, i32 %y) { -; CHECK-LABEL: sdiv_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %eax -; CHECK-NEXT: cltd -; CHECK-NEXT: idivl %esi -; CHECK-NEXT: testl %eax, %eax -; CHECK-NEXT: je .LBB35_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB35_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: sdiv_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: cltd +; X86-NEXT: idivl {{[0-9]+}}(%esp) +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB35_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB35_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: sdiv_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %eax +; X64-NEXT: cltd +; X64-NEXT: idivl %esi +; X64-NEXT: testl %eax, %eax +; X64-NEXT: je .LBB35_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB35_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = sdiv exact i32 %x, %y %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @add_known_nonzero(i32 %xx, i32 %y) { -; CHECK-LABEL: add_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: orl $1, %edi -; CHECK-NEXT: addl %esi, %edi -; CHECK-NEXT: rep bsfl %edi, %eax -; CHECK-NEXT: retq +; X86-LABEL: add_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: orl $1, %eax +; X86-NEXT: addl {{[0-9]+}}(%esp), %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: add_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: orl $1, %edi +; X64-NEXT: addl %esi, %edi +; X64-NEXT: rep bsfl %edi, %eax +; X64-NEXT: retq %x = or i32 %xx, 1 %z = add nuw i32 %x, %y %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -658,17 +1091,30 @@ define i32 @add_known_nonzero(i32 %xx, i32 %y) { } define i32 @add_maybe_zero(i32 %xx, i32 %y) { -; CHECK-LABEL: add_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: orl $1, %edi -; CHECK-NEXT: addl %esi, %edi -; CHECK-NEXT: je .LBB37_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %edi, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB37_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: add_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: orl $1, %eax +; X86-NEXT: addl {{[0-9]+}}(%esp), %eax +; X86-NEXT: je .LBB37_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB37_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: add_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: orl $1, %edi +; X64-NEXT: addl %esi, %edi +; X64-NEXT: je .LBB37_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %edi, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB37_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %x = or i32 %xx, 1 %z = add nsw i32 %x, %y %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -676,15 +1122,24 @@ define i32 @add_maybe_zero(i32 %xx, i32 %y) { } define i32 @sub_known_nonzero_neg_case(i32 %xx) { -; CHECK-LABEL: sub_known_nonzero_neg_case: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %ecx -; CHECK-NEXT: movl $256, %eax # imm = 0x100 -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: shll %cl, %eax -; CHECK-NEXT: negl %eax -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq +; X86-LABEL: sub_known_nonzero_neg_case: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $256, %eax # imm = 0x100 +; X86-NEXT: shll %cl, %eax +; X86-NEXT: negl %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: sub_known_nonzero_neg_case: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %ecx +; X64-NEXT: movl $256, %eax # imm = 0x100 +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: shll %cl, %eax +; X64-NEXT: negl %eax +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq %x = shl nuw nsw i32 256, %xx %z = sub i32 0, %x %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -692,14 +1147,24 @@ define i32 @sub_known_nonzero_neg_case(i32 %xx) { } define i32 @sub_known_nonzero_ne_case(i32 %xx, i32 %yy) { -; CHECK-LABEL: sub_known_nonzero_ne_case: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %eax -; CHECK-NEXT: orl $64, %eax -; CHECK-NEXT: andl $-65, %edi -; CHECK-NEXT: subl %eax, %edi -; CHECK-NEXT: rep bsfl %edi, %eax -; CHECK-NEXT: retq +; X86-LABEL: sub_known_nonzero_ne_case: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: movl %eax, %ecx +; X86-NEXT: orl $64, %ecx +; X86-NEXT: andl $-65, %eax +; X86-NEXT: subl %ecx, %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: sub_known_nonzero_ne_case: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %eax +; X64-NEXT: orl $64, %eax +; X64-NEXT: andl $-65, %edi +; X64-NEXT: subl %eax, %edi +; X64-NEXT: rep bsfl %edi, %eax +; X64-NEXT: retq %x = or i32 %xx, 64 %y = and i32 %xx, -65 %z = sub i32 %y, %x @@ -708,18 +1173,32 @@ define i32 @sub_known_nonzero_ne_case(i32 %xx, i32 %yy) { } define i32 @sub_maybe_zero(i32 %x) { -; CHECK-LABEL: sub_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %eax -; CHECK-NEXT: orl $64, %eax -; CHECK-NEXT: subl %edi, %eax -; CHECK-NEXT: je .LBB40_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB40_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: sub_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl %ecx, %eax +; X86-NEXT: orl $64, %eax +; X86-NEXT: subl %ecx, %eax +; X86-NEXT: je .LBB40_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB40_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: sub_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %eax +; X64-NEXT: orl $64, %eax +; X64-NEXT: subl %edi, %eax +; X64-NEXT: je .LBB40_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB40_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %y = or i32 %x, 64 %z = sub i32 %y, %x %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -727,34 +1206,60 @@ define i32 @sub_maybe_zero(i32 %x) { } define i32 @sub_maybe_zero2(i32 %x) { -; CHECK-LABEL: sub_maybe_zero2: -; CHECK: # %bb.0: -; CHECK-NEXT: negl %edi -; CHECK-NEXT: je .LBB41_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %edi, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB41_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: sub_maybe_zero2: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: negl %eax +; X86-NEXT: je .LBB41_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB41_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: sub_maybe_zero2: +; X64: # %bb.0: +; X64-NEXT: negl %edi +; X64-NEXT: je .LBB41_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %edi, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB41_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = sub i32 0, %x %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @mul_known_nonzero_nsw(i32 %x, i32 %yy) { -; CHECK-LABEL: mul_known_nonzero_nsw: -; CHECK: # %bb.0: -; CHECK-NEXT: orl $256, %esi # imm = 0x100 -; CHECK-NEXT: imull %edi, %esi -; CHECK-NEXT: testl %esi, %esi -; CHECK-NEXT: je .LBB42_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %esi, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB42_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: mul_known_nonzero_nsw: +; X86: # %bb.0: +; X86-NEXT: movl $256, %eax # imm = 0x100 +; X86-NEXT: orl {{[0-9]+}}(%esp), %eax +; X86-NEXT: imull {{[0-9]+}}(%esp), %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB42_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB42_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: mul_known_nonzero_nsw: +; X64: # %bb.0: +; X64-NEXT: orl $256, %esi # imm = 0x100 +; X64-NEXT: imull %edi, %esi +; X64-NEXT: testl %esi, %esi +; X64-NEXT: je .LBB42_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %esi, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB42_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %y = or i32 %yy, 256 %z = mul nsw i32 %y, %x %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -762,18 +1267,32 @@ define i32 @mul_known_nonzero_nsw(i32 %x, i32 %yy) { } define i32 @mul_known_nonzero_nuw(i32 %x, i32 %yy) { -; CHECK-LABEL: mul_known_nonzero_nuw: -; CHECK: # %bb.0: -; CHECK-NEXT: orl $256, %esi # imm = 0x100 -; CHECK-NEXT: imull %edi, %esi -; CHECK-NEXT: testl %esi, %esi -; CHECK-NEXT: je .LBB43_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %esi, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB43_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: mul_known_nonzero_nuw: +; X86: # %bb.0: +; X86-NEXT: movl $256, %eax # imm = 0x100 +; X86-NEXT: orl {{[0-9]+}}(%esp), %eax +; X86-NEXT: imull {{[0-9]+}}(%esp), %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB43_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB43_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: mul_known_nonzero_nuw: +; X64: # %bb.0: +; X64-NEXT: orl $256, %esi # imm = 0x100 +; X64-NEXT: imull %edi, %esi +; X64-NEXT: testl %esi, %esi +; X64-NEXT: je .LBB43_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %esi, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB43_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %y = or i32 %yy, 256 %z = mul nuw i32 %y, %x %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -781,36 +1300,63 @@ define i32 @mul_known_nonzero_nuw(i32 %x, i32 %yy) { } define i32 @mul_maybe_zero(i32 %x, i32 %y) { -; CHECK-LABEL: mul_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: imull %esi, %edi -; CHECK-NEXT: testl %edi, %edi -; CHECK-NEXT: je .LBB44_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %edi, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB44_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: mul_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: imull {{[0-9]+}}(%esp), %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB44_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB44_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: mul_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: imull %esi, %edi +; X64-NEXT: testl %edi, %edi +; X64-NEXT: je .LBB44_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %edi, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB44_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = mul nuw nsw i32 %y, %x %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @bitcast_known_nonzero(<2 x i16> %xx) { -; CHECK-LABEL: bitcast_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] -; CHECK-NEXT: pslld $23, %xmm0 -; CHECK-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-NEXT: cvttps2dq %xmm0, %xmm0 -; CHECK-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7] -; CHECK-NEXT: pmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-NEXT: movd %xmm0, %eax -; CHECK-NEXT: bsfl %eax, %ecx -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: cmovnel %ecx, %eax -; CHECK-NEXT: retq +; X86-LABEL: bitcast_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] +; X86-NEXT: pslld $23, %xmm0 +; X86-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0 +; X86-NEXT: cvttps2dq %xmm0, %xmm0 +; X86-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7] +; X86-NEXT: pmullw {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0 +; X86-NEXT: movd %xmm0, %eax +; X86-NEXT: bsfl %eax, %ecx +; X86-NEXT: movl $32, %eax +; X86-NEXT: cmovnel %ecx, %eax +; X86-NEXT: retl +; +; X64-LABEL: bitcast_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero +; X64-NEXT: vpslld $23, %xmm0, %xmm0 +; X64-NEXT: vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; X64-NEXT: vcvttps2dq %xmm0, %xmm0 +; X64-NEXT: vpackusdw %xmm0, %xmm0, %xmm0 +; X64-NEXT: vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; X64-NEXT: vmovd %xmm0, %eax +; X64-NEXT: bsfl %eax, %ecx +; X64-NEXT: movl $32, %eax +; X64-NEXT: cmovnel %ecx, %eax +; X64-NEXT: retq %x = shl nuw nsw <2 x i16> , %xx %z = bitcast <2 x i16> %x to i32 %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -818,49 +1364,83 @@ define i32 @bitcast_known_nonzero(<2 x i16> %xx) { } define i32 @bitcast_maybe_zero(<2 x i16> %x) { -; CHECK-LABEL: bitcast_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: movd %xmm0, %eax -; CHECK-NEXT: testl %eax, %eax -; CHECK-NEXT: je .LBB46_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB46_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: bitcast_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movd %xmm0, %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB46_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB46_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: bitcast_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: vmovd %xmm0, %eax +; X64-NEXT: testl %eax, %eax +; X64-NEXT: je .LBB46_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB46_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = bitcast <2 x i16> %x to i32 %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @bitcast_from_float(float %x) { -; CHECK-LABEL: bitcast_from_float: -; CHECK: # %bb.0: -; CHECK-NEXT: movd %xmm0, %eax -; CHECK-NEXT: testl %eax, %eax -; CHECK-NEXT: je .LBB47_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB47_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: bitcast_from_float: +; X86: # %bb.0: +; X86-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; X86-NEXT: movd %xmm0, %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB47_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB47_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: bitcast_from_float: +; X64: # %bb.0: +; X64-NEXT: vmovd %xmm0, %eax +; X64-NEXT: testl %eax, %eax +; X64-NEXT: je .LBB47_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB47_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = bitcast float %x to i32 %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @zext_known_nonzero(i16 %xx) { -; CHECK-LABEL: zext_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %ecx -; CHECK-NEXT: movl $256, %eax # imm = 0x100 -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: shll %cl, %eax -; CHECK-NEXT: movzwl %ax, %eax -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq +; X86-LABEL: zext_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $256, %eax # imm = 0x100 +; X86-NEXT: shll %cl, %eax +; X86-NEXT: movzwl %ax, %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: zext_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %ecx +; X64-NEXT: movl $256, %eax # imm = 0x100 +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: shll %cl, %eax +; X64-NEXT: movzwl %ax, %eax +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq %x = shl nuw nsw i16 256, %xx %z = zext i16 %x to i32 %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -868,32 +1448,54 @@ define i32 @zext_known_nonzero(i16 %xx) { } define i32 @zext_maybe_zero(i16 %x) { -; CHECK-LABEL: zext_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: testw %di, %di -; CHECK-NEXT: je .LBB49_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: movzwl %di, %eax -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB49_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: zext_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movzwl {{[0-9]+}}(%esp), %eax +; X86-NEXT: testw %ax, %ax +; X86-NEXT: je .LBB49_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: movzwl %ax, %eax +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB49_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: zext_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: testw %di, %di +; X64-NEXT: je .LBB49_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: movzwl %di, %eax +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB49_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = zext i16 %x to i32 %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r } define i32 @sext_known_nonzero(i16 %xx) { -; CHECK-LABEL: sext_known_nonzero: -; CHECK: # %bb.0: -; CHECK-NEXT: movl %edi, %ecx -; CHECK-NEXT: movl $256, %eax # imm = 0x100 -; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: shll %cl, %eax -; CHECK-NEXT: cwtl -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq +; X86-LABEL: sext_known_nonzero: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl $256, %eax # imm = 0x100 +; X86-NEXT: shll %cl, %eax +; X86-NEXT: cwtl +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; +; X64-LABEL: sext_known_nonzero: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %ecx +; X64-NEXT: movl $256, %eax # imm = 0x100 +; X64-NEXT: # kill: def $cl killed $cl killed $ecx +; X64-NEXT: shll %cl, %eax +; X64-NEXT: cwtl +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq %x = shl nuw nsw i16 256, %xx %z = sext i16 %x to i32 %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) @@ -901,17 +1503,29 @@ define i32 @sext_known_nonzero(i16 %xx) { } define i32 @sext_maybe_zero(i16 %x) { -; CHECK-LABEL: sext_maybe_zero: -; CHECK: # %bb.0: -; CHECK-NEXT: testw %di, %di -; CHECK-NEXT: je .LBB51_1 -; CHECK-NEXT: # %bb.2: # %cond.false -; CHECK-NEXT: movswl %di, %eax -; CHECK-NEXT: rep bsfl %eax, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB51_1: -; CHECK-NEXT: movl $32, %eax -; CHECK-NEXT: retq +; X86-LABEL: sext_maybe_zero: +; X86: # %bb.0: +; X86-NEXT: movswl {{[0-9]+}}(%esp), %eax +; X86-NEXT: testl %eax, %eax +; X86-NEXT: je .LBB51_1 +; X86-NEXT: # %bb.2: # %cond.false +; X86-NEXT: rep bsfl %eax, %eax +; X86-NEXT: retl +; X86-NEXT: .LBB51_1: +; X86-NEXT: movl $32, %eax +; X86-NEXT: retl +; +; X64-LABEL: sext_maybe_zero: +; X64: # %bb.0: +; X64-NEXT: testw %di, %di +; X64-NEXT: je .LBB51_1 +; X64-NEXT: # %bb.2: # %cond.false +; X64-NEXT: movswl %di, %eax +; X64-NEXT: rep bsfl %eax, %eax +; X64-NEXT: retq +; X64-NEXT: .LBB51_1: +; X64-NEXT: movl $32, %eax +; X64-NEXT: retq %z = sext i16 %x to i32 %r = call i32 @llvm.cttz.i32(i32 %z, i1 false) ret i32 %r -- GitLab From e8d5223ce4b2214e052b5b52b2e6453ffea0fe33 Mon Sep 17 00:00:00 2001 From: David Green Date: Sun, 24 Mar 2024 12:32:47 +0000 Subject: [PATCH 165/306] [AArch64] Additional GISel test coverage. NFC --- llvm/test/CodeGen/AArch64/setcc_knownbits.ll | 95 +++++++++++++------- 1 file changed, 65 insertions(+), 30 deletions(-) diff --git a/llvm/test/CodeGen/AArch64/setcc_knownbits.ll b/llvm/test/CodeGen/AArch64/setcc_knownbits.ll index bb9546af8bb7..6a337bf02e09 100644 --- a/llvm/test/CodeGen/AArch64/setcc_knownbits.ll +++ b/llvm/test/CodeGen/AArch64/setcc_knownbits.ll @@ -1,5 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3 -; RUN: llc < %s -mtriple=aarch64 | FileCheck %s +; RUN: llc < %s -mtriple=aarch64 | FileCheck %s --check-prefixes=CHECK,CHECK-SD +; RUN: llc < %s -mtriple=aarch64 -global-isel | FileCheck %s --check-prefixes=CHECK,CHECK-GI define i1 @load_bv_v4i8(i1 zeroext %a) { ; CHECK-LABEL: load_bv_v4i8: @@ -11,18 +12,31 @@ define i1 @load_bv_v4i8(i1 zeroext %a) { } define noundef i1 @logger(i32 noundef %logLevel, ptr %ea, ptr %pll) { -; CHECK-LABEL: logger: -; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ldr w8, [x2] -; CHECK-NEXT: cmp w8, w0 -; CHECK-NEXT: b.ls .LBB1_2 -; CHECK-NEXT: // %bb.1: -; CHECK-NEXT: mov w0, wzr -; CHECK-NEXT: ret -; CHECK-NEXT: .LBB1_2: // %land.rhs -; CHECK-NEXT: ldr x8, [x1] -; CHECK-NEXT: ldrb w0, [x8] -; CHECK-NEXT: ret +; CHECK-SD-LABEL: logger: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: ldr w8, [x2] +; CHECK-SD-NEXT: cmp w8, w0 +; CHECK-SD-NEXT: b.ls .LBB1_2 +; CHECK-SD-NEXT: // %bb.1: +; CHECK-SD-NEXT: mov w0, wzr +; CHECK-SD-NEXT: ret +; CHECK-SD-NEXT: .LBB1_2: // %land.rhs +; CHECK-SD-NEXT: ldr x8, [x1] +; CHECK-SD-NEXT: ldrb w0, [x8] +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: logger: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ldr w8, [x2] +; CHECK-GI-NEXT: cmp w8, w0 +; CHECK-GI-NEXT: mov w0, wzr +; CHECK-GI-NEXT: b.hi .LBB1_2 +; CHECK-GI-NEXT: // %bb.1: // %land.rhs +; CHECK-GI-NEXT: ldr x8, [x1] +; CHECK-GI-NEXT: ldrb w8, [x8] +; CHECK-GI-NEXT: and w0, w8, #0x1 +; CHECK-GI-NEXT: .LBB1_2: // %land.end +; CHECK-GI-NEXT: ret entry: %0 = load i32, ptr %pll, align 4 %cmp.not = icmp ugt i32 %0, %logLevel @@ -44,12 +58,20 @@ land.end: ; preds = %land.rhs, %entry declare i64 @llvm.ctlz.i64(i64 %in, i1) define i1 @lshr_ctlz_undef_cmpeq_one_i64(i64 %in) { -; CHECK-LABEL: lshr_ctlz_undef_cmpeq_one_i64: -; CHECK: // %bb.0: -; CHECK-NEXT: clz x8, x0 -; CHECK-NEXT: lsr x0, x8, #6 -; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0 -; CHECK-NEXT: ret +; CHECK-SD-LABEL: lshr_ctlz_undef_cmpeq_one_i64: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: clz x8, x0 +; CHECK-SD-NEXT: lsr x0, x8, #6 +; CHECK-SD-NEXT: // kill: def $w0 killed $w0 killed $x0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: lshr_ctlz_undef_cmpeq_one_i64: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: clz x8, x0 +; CHECK-GI-NEXT: lsr x8, x8, #6 +; CHECK-GI-NEXT: cmp x8, #1 +; CHECK-GI-NEXT: cset w0, eq +; CHECK-GI-NEXT: ret %ctlz = call i64 @llvm.ctlz.i64(i64 %in, i1 -1) %lshr = lshr i64 %ctlz, 6 %icmp = icmp eq i64 %lshr, 1 @@ -57,17 +79,30 @@ define i1 @lshr_ctlz_undef_cmpeq_one_i64(i64 %in) { } define i32 @PR17487(i1 %tobool) { -; CHECK-LABEL: PR17487: -; CHECK: // %bb.0: -; CHECK-NEXT: dup v0.2s, w0 -; CHECK-NEXT: mov w8, #1 // =0x1 -; CHECK-NEXT: dup v1.2d, x8 -; CHECK-NEXT: ushll v0.2d, v0.2s, #0 -; CHECK-NEXT: bic v0.16b, v1.16b, v0.16b -; CHECK-NEXT: mov x8, v0.d[1] -; CHECK-NEXT: cmp x8, #1 -; CHECK-NEXT: cset w0, ne -; CHECK-NEXT: ret +; CHECK-SD-LABEL: PR17487: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: dup v0.2s, w0 +; CHECK-SD-NEXT: mov w8, #1 // =0x1 +; CHECK-SD-NEXT: dup v1.2d, x8 +; CHECK-SD-NEXT: ushll v0.2d, v0.2s, #0 +; CHECK-SD-NEXT: bic v0.16b, v1.16b, v0.16b +; CHECK-SD-NEXT: mov x8, v0.d[1] +; CHECK-SD-NEXT: cmp x8, #1 +; CHECK-SD-NEXT: cset w0, ne +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: PR17487: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: // kill: def $w0 killed $w0 def $x0 +; CHECK-GI-NEXT: mov v0.d[1], x0 +; CHECK-GI-NEXT: adrp x8, .LCPI3_0 +; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI3_0] +; CHECK-GI-NEXT: bic v0.16b, v1.16b, v0.16b +; CHECK-GI-NEXT: mov d0, v0.d[1] +; CHECK-GI-NEXT: fmov x8, d0 +; CHECK-GI-NEXT: cmp x8, #1 +; CHECK-GI-NEXT: cset w0, ne +; CHECK-GI-NEXT: ret %tmp = insertelement <2 x i1> undef, i1 %tobool, i32 1 %tmp1 = zext <2 x i1> %tmp to <2 x i64> %tmp2 = xor <2 x i64> %tmp1, -- GitLab From b3fe27f2be0585d5d2ad46f96956ccfd76ca003e Mon Sep 17 00:00:00 2001 From: Marc Auberer Date: Sun, 24 Mar 2024 16:14:56 +0100 Subject: [PATCH 166/306] [InstCombine] Copy flags of extractelement for extelt -> icmp combine (#86366) Fixes #86164 --- llvm/include/llvm/IR/InstrTypes.h | 11 +++++++++++ llvm/lib/IR/Instructions.cpp | 10 ++++++++++ .../Transforms/InstCombine/InstCombineVectorOps.cpp | 4 +++- llvm/test/Transforms/InstCombine/scalarization.ll | 11 +++++++++++ 4 files changed, 35 insertions(+), 1 deletion(-) diff --git a/llvm/include/llvm/IR/InstrTypes.h b/llvm/include/llvm/IR/InstrTypes.h index e8c2cba8418d..d2b33fe9c651 100644 --- a/llvm/include/llvm/IR/InstrTypes.h +++ b/llvm/include/llvm/IR/InstrTypes.h @@ -1058,6 +1058,17 @@ public: static CmpInst *Create(OtherOps Op, Predicate predicate, Value *S1, Value *S2, const Twine &Name, BasicBlock *InsertAtEnd); + /// Construct a compare instruction, given the opcode, the predicate, + /// the two operands and the instruction to copy the flags from. Optionally + /// (if InstBefore is specified) insert the instruction into a BasicBlock + /// right before the specified instruction. The specified Instruction is + /// allowed to be a dereferenced end iterator. Create a CmpInst + static CmpInst *CreateWithCopiedFlags(OtherOps Op, Predicate Pred, Value *S1, + Value *S2, + const Instruction *FlagsSource, + const Twine &Name = "", + Instruction *InsertBefore = nullptr); + /// Get the opcode casted to the right type OtherOps getOpcode() const { return static_cast(Instruction::getOpcode()); diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp index 494d50f89e37..8ca211e6e2e7 100644 --- a/llvm/lib/IR/Instructions.cpp +++ b/llvm/lib/IR/Instructions.cpp @@ -4623,6 +4623,16 @@ CmpInst::Create(OtherOps Op, Predicate predicate, Value *S1, Value *S2, S1, S2, Name); } +CmpInst *CmpInst::CreateWithCopiedFlags(OtherOps Op, Predicate Pred, Value *S1, + Value *S2, + const Instruction *FlagsSource, + const Twine &Name, + Instruction *InsertBefore) { + CmpInst *Inst = Create(Op, Pred, S1, S2, Name, InsertBefore); + Inst->copyIRFlags(FlagsSource); + return Inst; +} + void CmpInst::swapOperands() { if (ICmpInst *IC = dyn_cast(this)) IC->swapOperands(); diff --git a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp index c7f4fb17648c..99f1f8eb34bb 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp @@ -487,7 +487,9 @@ Instruction *InstCombinerImpl::visitExtractElementInst(ExtractElementInst &EI) { // extelt (cmp X, Y), Index --> cmp (extelt X, Index), (extelt Y, Index) Value *E0 = Builder.CreateExtractElement(X, Index); Value *E1 = Builder.CreateExtractElement(Y, Index); - return CmpInst::Create(cast(SrcVec)->getOpcode(), Pred, E0, E1); + CmpInst *SrcCmpInst = cast(SrcVec); + return CmpInst::CreateWithCopiedFlags(SrcCmpInst->getOpcode(), Pred, E0, E1, + SrcCmpInst); } if (auto *I = dyn_cast(SrcVec)) { diff --git a/llvm/test/Transforms/InstCombine/scalarization.ll b/llvm/test/Transforms/InstCombine/scalarization.ll index fe6dc526bd50..7e645ef7e883 100644 --- a/llvm/test/Transforms/InstCombine/scalarization.ll +++ b/llvm/test/Transforms/InstCombine/scalarization.ll @@ -341,6 +341,17 @@ define i1 @extractelt_vector_fcmp_constrhs_dynidx(<2 x float> %arg, i32 %idx) { ret i1 %ext } +define i1 @extractelt_vector_fcmp_copy_flags(<4 x float> %x) { +; CHECK-LABEL: @extractelt_vector_fcmp_copy_flags( +; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x float> [[X:%.*]], i64 2 +; CHECK-NEXT: [[TMP2:%.*]] = fcmp nsz arcp oeq float [[TMP1]], 0.000000e+00 +; CHECK-NEXT: ret i1 [[TMP2]] +; + %cmp = fcmp nsz arcp oeq <4 x float> %x, zeroinitializer + %r = extractelement <4 x i1> %cmp, i32 2 + ret i1 %r +} + define i1 @extractelt_vector_fcmp_not_cheap_to_scalarize_multi_use(<2 x float> %arg0, <2 x float> %arg1, <2 x float> %arg2, i32 %idx) { ; ; CHECK-LABEL: @extractelt_vector_fcmp_not_cheap_to_scalarize_multi_use( -- GitLab From 9632e1515c93453efc39752b1c9f32aedd358fbc Mon Sep 17 00:00:00 2001 From: houndlord <45179481+houndlord@users.noreply.github.com> Date: Sun, 24 Mar 2024 16:33:16 +0100 Subject: [PATCH 167/306] Match fixed width ISD::AVGFLOORS + ISD::AVGCEILS patterns (#86222) --- llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 37 ++++++++++++++----- llvm/test/CodeGen/AArch64/hadd-combine.ll | 24 ++++++++++++ 2 files changed, 51 insertions(+), 10 deletions(-) diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index e27a8bb8fdac..05b4ce3aaa2c 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -2529,20 +2529,28 @@ static SDValue foldAddSubBoolOfMaskedVal(SDNode *N, SelectionDAG &DAG) { return DAG.getNode(IsAdd ? ISD::SUB : ISD::ADD, DL, VT, C1, LowBit); } -// Attempt to form avgceilu(A, B) from (A | B) - ((A ^ B) >> 1) -static SDValue combineFixedwidthToAVGCEILU(SDNode *N, SelectionDAG &DAG) { +// Attempt to form avgceil(A, B) from (A | B) - ((A ^ B) >> 1) +static SDValue combineFixedwidthToAVGCEIL(SDNode *N, SelectionDAG &DAG) { const TargetLowering &TLI = DAG.getTargetLoweringInfo(); SDValue N0 = N->getOperand(0); EVT VT = N0.getValueType(); SDLoc DL(N); + SDValue A, B; + if (TLI.isOperationLegal(ISD::AVGCEILU, VT)) { - SDValue A, B; if (sd_match(N, m_Sub(m_Or(m_Value(A), m_Value(B)), m_Srl(m_Xor(m_Deferred(A), m_Deferred(B)), m_SpecificInt(1))))) { return DAG.getNode(ISD::AVGCEILU, DL, VT, A, B); } } + if (TLI.isOperationLegal(ISD::AVGCEILS, VT)) { + if (sd_match(N, m_Sub(m_Or(m_Value(A), m_Value(B)), + m_Sra(m_Xor(m_Deferred(A), m_Deferred(B)), + m_SpecificInt(1))))) { + return DAG.getNode(ISD::AVGCEILS, DL, VT, A, B); + } + } return SDValue(); } @@ -2837,20 +2845,29 @@ SDValue DAGCombiner::visitADDLike(SDNode *N) { return SDValue(); } -// Attempt to form avgflooru(A, B) from (A & B) + ((A ^ B) >> 1) -static SDValue combineFixedwidthToAVGFLOORU(SDNode *N, SelectionDAG &DAG) { +// Attempt to form avgfloor(A, B) from (A & B) + ((A ^ B) >> 1) +static SDValue combineFixedwidthToAVGFLOOR(SDNode *N, SelectionDAG &DAG) { const TargetLowering &TLI = DAG.getTargetLoweringInfo(); SDValue N0 = N->getOperand(0); EVT VT = N0.getValueType(); SDLoc DL(N); + SDValue A, B; + if (TLI.isOperationLegal(ISD::AVGFLOORU, VT)) { - SDValue A, B; if (sd_match(N, m_Add(m_And(m_Value(A), m_Value(B)), m_Srl(m_Xor(m_Deferred(A), m_Deferred(B)), m_SpecificInt(1))))) { return DAG.getNode(ISD::AVGFLOORU, DL, VT, A, B); } } + if (TLI.isOperationLegal(ISD::AVGFLOORS, VT)) { + if (sd_match(N, m_Add(m_And(m_Value(A), m_Value(B)), + m_Sra(m_Xor(m_Deferred(A), m_Deferred(B)), + m_SpecificInt(1))))) { + return DAG.getNode(ISD::AVGFLOORS, DL, VT, A, B); + } + } + return SDValue(); } @@ -2869,8 +2886,8 @@ SDValue DAGCombiner::visitADD(SDNode *N) { if (SDValue V = foldAddSubOfSignBit(N, DAG)) return V; - // Try to match AVGFLOORU fixedwidth pattern - if (SDValue V = combineFixedwidthToAVGFLOORU(N, DAG)) + // Try to match AVGFLOOR fixedwidth pattern + if (SDValue V = combineFixedwidthToAVGFLOOR(N, DAG)) return V; // fold (a+b) -> (a|b) iff a and b share no bits. @@ -3868,8 +3885,8 @@ SDValue DAGCombiner::visitSUB(SDNode *N) { if (SDValue V = foldAddSubOfSignBit(N, DAG)) return V; - // Try to match AVGCEILU fixedwidth pattern - if (SDValue V = combineFixedwidthToAVGCEILU(N, DAG)) + // Try to match AVGCEIL fixedwidth pattern + if (SDValue V = combineFixedwidthToAVGCEIL(N, DAG)) return V; if (SDValue V = foldAddSubMasked1(false, N0, N1, DAG, SDLoc(N))) diff --git a/llvm/test/CodeGen/AArch64/hadd-combine.ll b/llvm/test/CodeGen/AArch64/hadd-combine.ll index e12502980790..491bf40ea4aa 100644 --- a/llvm/test/CodeGen/AArch64/hadd-combine.ll +++ b/llvm/test/CodeGen/AArch64/hadd-combine.ll @@ -341,6 +341,18 @@ define <8 x i16> @sub_fixedwidth_v4i32(<8 x i16> %a0, <8 x i16> %a1) { ret <8 x i16> %res } +define <8 x i16> @srhadd_fixedwidth_v8i16(<8 x i16> %a0, <8 x i16> %a1) { +; CHECK-LABEL: srhadd_fixedwidth_v8i16: +; CHECK: // %bb.0: +; CHECK-NEXT: srhadd v0.8h, v0.8h, v1.8h +; CHECK-NEXT: ret + %or = or <8 x i16> %a0, %a1 + %xor = xor <8 x i16> %a0, %a1 + %srl = ashr <8 x i16> %xor, + %res = sub <8 x i16> %or, %srl + ret <8 x i16> %res +} + define <8 x i16> @rhaddu_base(<8 x i16> %src1, <8 x i16> %src2) { ; CHECK-LABEL: rhaddu_base: ; CHECK: // %bb.0: @@ -879,6 +891,18 @@ define <8 x i16> @uhadd_fixedwidth_v4i32(<8 x i16> %a0, <8 x i16> %a1) { ret <8 x i16> %res } +define <8 x i16> @shadd_fixedwidth_v8i16(<8 x i16> %a0, <8 x i16> %a1) { +; CHECK-LABEL: shadd_fixedwidth_v8i16: +; CHECK: // %bb.0: +; CHECK-NEXT: shadd v0.8h, v0.8h, v1.8h +; CHECK-NEXT: ret + %and = and <8 x i16> %a0, %a1 + %xor = xor <8 x i16> %a0, %a1 + %srl = ashr <8 x i16> %xor, + %res = add <8 x i16> %and, %srl + ret <8 x i16> %res +} + declare <8 x i8> @llvm.aarch64.neon.shadd.v8i8(<8 x i8>, <8 x i8>) declare <4 x i16> @llvm.aarch64.neon.shadd.v4i16(<4 x i16>, <4 x i16>) declare <2 x i32> @llvm.aarch64.neon.shadd.v2i32(<2 x i32>, <2 x i32>) -- GitLab From 48048051323d5dd74057dc5f32df8c3c323afcd5 Mon Sep 17 00:00:00 2001 From: Arthur Eubanks Date: Sun, 24 Mar 2024 11:43:15 -0600 Subject: [PATCH 168/306] [lld/ELF][X86] Respect outSecOff when checking if GOTPCREL can be relaxed (#86334) The existing implementation didn't handle when the input text section was some offset from the output section. This resulted in an assert in relaxGot() with an lld built with asserts for some large binaries, or even worse, a silently broken binary with an lld without asserts. --- lld/ELF/Arch/X86_64.cpp | 7 ++++--- lld/test/ELF/x86-64-gotpc-relax-too-far.s | 12 +++++++++++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/lld/ELF/Arch/X86_64.cpp b/lld/ELF/Arch/X86_64.cpp index de459013595f..a85bf3aa0c09 100644 --- a/lld/ELF/Arch/X86_64.cpp +++ b/lld/ELF/Arch/X86_64.cpp @@ -328,9 +328,10 @@ bool X86_64::relaxOnce(int pass) const { if (rel.expr != R_RELAX_GOT_PC) continue; - uint64_t v = sec->getRelocTargetVA( - sec->file, rel.type, rel.addend, - sec->getOutputSection()->addr + rel.offset, *rel.sym, rel.expr); + uint64_t v = sec->getRelocTargetVA(sec->file, rel.type, rel.addend, + sec->getOutputSection()->addr + + sec->outSecOff + rel.offset, + *rel.sym, rel.expr); if (isInt<32>(v)) continue; if (rel.sym->auxIdx == 0) { diff --git a/lld/test/ELF/x86-64-gotpc-relax-too-far.s b/lld/test/ELF/x86-64-gotpc-relax-too-far.s index 74aa6d8f65a0..ba41faab67de 100644 --- a/lld/test/ELF/x86-64-gotpc-relax-too-far.s +++ b/lld/test/ELF/x86-64-gotpc-relax-too-far.s @@ -5,7 +5,10 @@ # RUN: llvm-objdump --no-print-imm-hex -d %t/bin | FileCheck --check-prefix=DISASM %s # RUN: llvm-readelf -S %t/bin | FileCheck --check-prefixes=GOT %s # RUN: ld.lld -T %t/lds2 %t/a.o -o %t/bin2 -# RUN: llvm-readelf -S %t/bin2 | FileCheck --check-prefixes=UNNECESSARY-GOT %s +# RUN: llvm-objdump --no-print-imm-hex -d %t/bin2 | FileCheck --check-prefix=DISASM %s +# RUN: llvm-readelf -S %t/bin2 | FileCheck --check-prefixes=GOT %s +# RUN: ld.lld -T %t/lds3 %t/a.o -o %t/bin3 +# RUN: llvm-readelf -S %t/bin3 | FileCheck --check-prefixes=UNNECESSARY-GOT %s # DISASM: <_foo>: # DISASM-NEXT: movl 2097146(%rip), %eax @@ -47,6 +50,13 @@ SECTIONS { data 0x80200000 : { *(data) } } #--- lds2 +SECTIONS { + .text.foo 0x100000 : { *(.text.foo) } + .text 0x1ff000 : { . = . + 0x1000 ; *(.text) } + .got 0x300000 : { *(.got) } + data 0x80200000 : { *(data) } +} +#--- lds3 SECTIONS { .text.foo 0x100000 : { *(.text.foo) } .text 0x200000 : { *(.text) } -- GitLab From d4a4585165f5c6ca8c42920b70e1b47696ff1172 Mon Sep 17 00:00:00 2001 From: Piotr Zegar Date: Sun, 24 Mar 2024 19:52:19 +0100 Subject: [PATCH 169/306] [clang-tidy] Ignore expresions in unevaluated context in bugprone-inc-dec-in-conditions (#85849) Skip checking for references to variable in unevaluated context, like decltype, static_assert and so on. Closes #85838 --- .../clang-tidy/bugprone/IncDecInConditionsCheck.cpp | 8 +++++++- clang-tools-extra/docs/ReleaseNotes.rst | 4 ++++ .../checkers/bugprone/inc-dec-in-conditions.cpp | 10 ++++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/clang-tools-extra/clang-tidy/bugprone/IncDecInConditionsCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/IncDecInConditionsCheck.cpp index 16f43128d55e..9b3b01eb0268 100644 --- a/clang-tools-extra/clang-tidy/bugprone/IncDecInConditionsCheck.cpp +++ b/clang-tools-extra/clang-tidy/bugprone/IncDecInConditionsCheck.cpp @@ -31,6 +31,10 @@ void IncDecInConditionsCheck::registerMatchers(MatchFinder *Finder) { anyOf(binaryOperator(anyOf(isComparisonOperator(), isLogicalOperator())), cxxOperatorCallExpr(isComparisonOperator()))); + auto IsInUnevaluatedContext = + expr(anyOf(hasAncestor(expr(matchers::hasUnevaluatedContext())), + hasAncestor(typeLoc()))); + Finder->addMatcher( expr( OperatorMatcher, unless(isExpansionInSystemHeader()), @@ -42,12 +46,14 @@ void IncDecInConditionsCheck::registerMatchers(MatchFinder *Finder) { cxxOperatorCallExpr( isPrePostOperator(), hasUnaryOperand(expr().bind("operand")))), + unless(IsInUnevaluatedContext), hasAncestor( expr(equalsBoundNode("parent"), hasDescendant( expr(unless(equalsBoundNode("operand")), matchers::isStatementIdenticalToBoundNode( - "operand")) + "operand"), + unless(IsInUnevaluatedContext)) .bind("second"))))) .bind("operator"))), this); diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst index a604e9276668..2392ccaf6575 100644 --- a/clang-tools-extra/docs/ReleaseNotes.rst +++ b/clang-tools-extra/docs/ReleaseNotes.rst @@ -139,6 +139,10 @@ Changes in existing checks ` check by detecting side effect from calling a method with non-const reference parameters. +- Improved :doc:`bugprone-inc-dec-in-conditions + ` check to ignore code + within unevaluated contexts, such as ``decltype``. + - Improved :doc:`bugprone-non-zero-enum-to-bool-conversion ` check by eliminating false positives resulting from direct usage of bitwise operators diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/inc-dec-in-conditions.cpp b/clang-tools-extra/test/clang-tidy/checkers/bugprone/inc-dec-in-conditions.cpp index 82af039973c3..91de013138f0 100644 --- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/inc-dec-in-conditions.cpp +++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/inc-dec-in-conditions.cpp @@ -68,3 +68,13 @@ bool doubleCheck(Container x) { // CHECK-MESSAGES: :[[@LINE-1]]:11: warning: decrementing and referencing a variable in a complex condition can cause unintended side-effects due to C++'s order of evaluation, consider moving the modification outside of the condition to avoid misunderstandings [bugprone-inc-dec-in-conditions] // CHECK-MESSAGES: :[[@LINE-2]]:31: warning: incrementing and referencing a variable in a complex condition can cause unintended side-effects due to C++'s order of evaluation, consider moving the modification outside of the condition to avoid misunderstandings [bugprone-inc-dec-in-conditions] } + +namespace PR85838 { + void test() + { + auto foo = 0; + auto bar = 0; + if (++foo < static_cast(bar)) {} + if (static_cast(bar) < foo) {} + } +} -- GitLab From 909ea28ac60760a0e9b39369b2f54a4e8f1daec9 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Sun, 24 Mar 2024 14:29:23 -0500 Subject: [PATCH 170/306] [Libomptarget] Specificall add LLVM include dirs in plugins --- openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt index 0420d0e6f1f8..b84f3d7b137c 100644 --- a/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt +++ b/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt @@ -60,6 +60,7 @@ target_link_options(PluginCommon PUBLIC ${offload_link_flags}) target_include_directories(PluginCommon PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include + ${LIBOMPTARGET_LLVM_INCLUDE_DIRS} ${LIBOMPTARGET_INCLUDE_DIR} ) -- GitLab From 488a18738f5c275093bcc5459da69e9b4c9de074 Mon Sep 17 00:00:00 2001 From: Lang Hames Date: Sun, 24 Mar 2024 10:41:21 -0500 Subject: [PATCH 171/306] [JITLink][ELF] Improve ELF section start/end symbol handling. This commit adds section start and stop symbol handling to ELF/aarch64, and fixes the section symbol prefixes (using `__start_` and `__stop_`, rather than `__start` and `__end`). It also adds a testcase for handling of these symbols. --- .../llvm/ExecutionEngine/JITLink/JITLink.h | 2 +- .../DefineExternalSectionStartAndEndSymbols.h | 19 ++++++++ .../ExecutionEngine/JITLink/ELF_aarch64.cpp | 7 +++ .../ExecutionEngine/JITLink/ELF_x86_64.cpp | 18 -------- .../ELF_section_start_and_stop_symbols.s | 43 +++++++++++++++++++ 5 files changed, 70 insertions(+), 19 deletions(-) create mode 100644 llvm/test/ExecutionEngine/JITLink/AArch64/ELF_section_start_and_stop_symbols.s diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h b/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h index 30a9383dc0bc..4ce0c8d9fd17 100644 --- a/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h @@ -567,7 +567,7 @@ public: orc::ExecutorAddrDiff getOffset() const { return Offset; } void setOffset(orc::ExecutorAddrDiff NewOffset) { - assert(NewOffset < getBlock().getSize() && "Offset out of range"); + assert(NewOffset <= getBlock().getSize() && "Offset out of range"); Offset = NewOffset; } diff --git a/llvm/lib/ExecutionEngine/JITLink/DefineExternalSectionStartAndEndSymbols.h b/llvm/lib/ExecutionEngine/JITLink/DefineExternalSectionStartAndEndSymbols.h index 159880e4b152..aef441d093a2 100644 --- a/llvm/lib/ExecutionEngine/JITLink/DefineExternalSectionStartAndEndSymbols.h +++ b/llvm/lib/ExecutionEngine/JITLink/DefineExternalSectionStartAndEndSymbols.h @@ -108,6 +108,25 @@ createDefineExternalSectionStartAndEndSymbolsPass( std::forward(F)); } +/// ELF section start/end symbol detection. +inline SectionRangeSymbolDesc +identifyELFSectionStartAndEndSymbols(LinkGraph &G, Symbol &Sym) { + constexpr StringRef StartSymbolPrefix = "__start_"; + constexpr StringRef EndSymbolPrefix = "__stop_"; + + auto SymName = Sym.getName(); + if (SymName.starts_with(StartSymbolPrefix)) { + if (auto *Sec = + G.findSectionByName(SymName.drop_front(StartSymbolPrefix.size()))) + return {*Sec, true}; + } else if (SymName.starts_with(EndSymbolPrefix)) { + if (auto *Sec = + G.findSectionByName(SymName.drop_front(EndSymbolPrefix.size()))) + return {*Sec, false}; + } + return {}; +} + } // end namespace jitlink } // end namespace llvm diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp index f17b2c626ac2..4f64d83162b2 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp @@ -20,6 +20,8 @@ #include "llvm/Object/ELFObjectFile.h" #include "llvm/Support/Endian.h" +#include "DefineExternalSectionStartAndEndSymbols.h" + #define DEBUG_TYPE "jitlink" using namespace llvm; @@ -611,6 +613,11 @@ void link_ELF_aarch64(std::unique_ptr G, else Config.PrePrunePasses.push_back(markAllSymbolsLive); + // Resolve any external section start / end symbols. + Config.PostAllocationPasses.push_back( + createDefineExternalSectionStartAndEndSymbolsPass( + identifyELFSectionStartAndEndSymbols)); + // Add an in-place GOT/TLS/Stubs build pass. Config.PostPrunePasses.push_back(buildTables_ELF_aarch64); } diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp index a1fe9c5fcd73..52dd83d9040f 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp @@ -343,24 +343,6 @@ createLinkGraphFromELFObject_x86_64(MemoryBufferRef ObjectBuffer) { .buildGraph(); } -static SectionRangeSymbolDesc -identifyELFSectionStartAndEndSymbols(LinkGraph &G, Symbol &Sym) { - constexpr StringRef StartSymbolPrefix = "__start"; - constexpr StringRef EndSymbolPrefix = "__end"; - - auto SymName = Sym.getName(); - if (SymName.starts_with(StartSymbolPrefix)) { - if (auto *Sec = - G.findSectionByName(SymName.drop_front(StartSymbolPrefix.size()))) - return {*Sec, true}; - } else if (SymName.starts_with(EndSymbolPrefix)) { - if (auto *Sec = - G.findSectionByName(SymName.drop_front(EndSymbolPrefix.size()))) - return {*Sec, false}; - } - return {}; -} - void link_ELF_x86_64(std::unique_ptr G, std::unique_ptr Ctx) { PassConfiguration Config; diff --git a/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_section_start_and_stop_symbols.s b/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_section_start_and_stop_symbols.s new file mode 100644 index 000000000000..f8e7ba96f006 --- /dev/null +++ b/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_section_start_and_stop_symbols.s @@ -0,0 +1,43 @@ +# RUN: llvm-mc -triple=aarch64-unknown-linux-gnu -position-independent \ +# RUN: -filetype=obj -o %t.o %s +# RUN: llvm-jitlink -noexec -check %s %t.o + + .text + .file "elf_section_start_stop.c" + .globl main + .p2align 2 + .type main,@function +main: + adrp x8, z + adrp x9, y + ldr w8, [x8, :lo12:z] + ldr w9, [x9, :lo12:y] + sub w0, w8, w9 + ret +.Lfunc_end0: + .size main, .Lfunc_end0-main + + .type x,@object + .section custom_section,"aw",@progbits + .globl x + .p2align 2 +x: + .word 42 + .size x, 4 + +# jitlink-check: *{8}z = (*{8}y) + 4 + + .type y,@object + .data + .globl y + .p2align 3, 0x0 +y: + .xword __start_custom_section + .size y, 8 + + .type z,@object + .globl z + .p2align 3, 0x0 +z: + .xword __stop_custom_section + .size z, 8 -- GitLab From 9f0321ccf118b37e5cb93cabd2acbf600c36b6ee Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Sun, 24 Mar 2024 15:23:50 -0500 Subject: [PATCH 172/306] [Libomptarget] Make plugins depend explicitly on `intrinsics_gen` Summary: It's possible for the OpenMP offloading plugins to be build before tablegen is run despite the fact that we rely on it. Simply make it depend on it currently. --- openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt index b84f3d7b137c..a7350e662a7c 100644 --- a/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt +++ b/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt @@ -19,6 +19,7 @@ add_library(PluginCommon OBJECT src/RPC.cpp src/Utils/ELF.cpp ) +add_dependencies(PluginCommon intrinsics_gen) # Only enable JIT for those targets that LLVM can support. string(TOUPPER "${LLVM_TARGETS_TO_BUILD}" TargetsSupported) -- GitLab From 81e2693c1202d6c4e48dbf2d985153b03cfccb79 Mon Sep 17 00:00:00 2001 From: Hui Date: Sun, 24 Mar 2024 20:52:47 +0000 Subject: [PATCH 173/306] [libc++][test] Fix race condition in condition_variable_any tests (#84788) Some tests in `condition_variable_any` use two `shared_lock` to guard, which does not work. The fix is to make the writer to use `unique_lock` --- .../thread.condition.condvarany/wait_for_token_pred.pass.cpp | 2 +- .../thread.condition.condvarany/wait_token_pred.pass.cpp | 2 +- .../thread.condition.condvarany/wait_until_token_pred.pass.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_for_token_pred.pass.cpp b/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_for_token_pred.pass.cpp index 4ea60557d9f8..7a39d1253a33 100644 --- a/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_for_token_pred.pass.cpp +++ b/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_for_token_pred.pass.cpp @@ -119,7 +119,7 @@ void test() { bool flag = false; auto thread = support::make_test_thread([&]() { std::this_thread::sleep_for(2ms); - Lock lock2{mutex}; + std::unique_lock lock2{mutex}; flag = true; cv.notify_all(); }); diff --git a/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_token_pred.pass.cpp b/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_token_pred.pass.cpp index e96a3e8bd1bc..f322d8cfdc68 100644 --- a/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_token_pred.pass.cpp +++ b/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_token_pred.pass.cpp @@ -63,7 +63,7 @@ void test() { bool flag = false; auto thread = support::make_test_thread([&]() { std::this_thread::sleep_for(std::chrono::milliseconds(2)); - Lock lock2{mutex}; + std::unique_lock lock2{mutex}; flag = true; cv.notify_all(); }); diff --git a/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_until_token_pred.pass.cpp b/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_until_token_pred.pass.cpp index d649db025d75..e7388b9ce0e1 100644 --- a/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_until_token_pred.pass.cpp +++ b/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_until_token_pred.pass.cpp @@ -119,7 +119,7 @@ void test() { bool flag = false; auto thread = support::make_test_thread([&]() { std::this_thread::sleep_for(std::chrono::milliseconds(2)); - Lock lock2{mutex}; + std::unique_lock lock2{mutex}; flag = true; cv.notify_all(); }); -- GitLab From f0a8738401137c32f3e1b9799244ed643feb7bf0 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Sun, 24 Mar 2024 20:59:53 +0000 Subject: [PATCH 174/306] [VPlan] Generate CalculateTripCountMinusVF for Part 0 only. (NFCI). The value produced by CalculateTripCountMinusVF VPInstructions is independent of the part. Only compute it for part 0 and use that for other parts. --- .../lib/Transforms/Vectorize/VPlanRecipes.cpp | 3 + .../AArch64/scalable-strict-fadd.ll | 63 +++---------------- .../AArch64/sve-tail-folding-unroll.ll | 42 ++----------- .../AArch64/uniform-args-call-variants.ll | 13 +--- 4 files changed, 21 insertions(+), 100 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index d75e322a74cf..f91997a37993 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -345,6 +345,9 @@ Value *VPInstruction::generateInstruction(VPTransformState &State, return Builder.CreateVectorSplice(PartMinus1, V2, -1, Name); } case VPInstruction::CalculateTripCountMinusVF: { + if (Part != 0) + return State.get(this, 0, /*IsScalar*/ true); + Value *ScalarTC = State.get(getOperand(0), {0, 0}); Value *Step = createStepForVF(Builder, ScalarTC->getType(), State.VF, State.UF); diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll index fc67fb5aded6..ad6e8534f318 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll @@ -403,21 +403,6 @@ define float @fadd_strict_unroll(ptr noalias nocapture readonly %a, i64 %n) #0 { ; CHECK-ORDERED-TF-NEXT: [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]] ; CHECK-ORDERED-TF-NEXT: [[TMP8:%.*]] = icmp ugt i64 [[N]], [[TMP6]] ; CHECK-ORDERED-TF-NEXT: [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0 -; CHECK-ORDERED-TF-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-TF-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 32 -; CHECK-ORDERED-TF-NEXT: [[TMP12:%.*]] = sub i64 [[N]], [[TMP11]] -; CHECK-ORDERED-TF-NEXT: [[TMP13:%.*]] = icmp ugt i64 [[N]], [[TMP11]] -; CHECK-ORDERED-TF-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 0 -; CHECK-ORDERED-TF-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-TF-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 32 -; CHECK-ORDERED-TF-NEXT: [[TMP17:%.*]] = sub i64 [[N]], [[TMP16]] -; CHECK-ORDERED-TF-NEXT: [[TMP18:%.*]] = icmp ugt i64 [[N]], [[TMP16]] -; CHECK-ORDERED-TF-NEXT: [[TMP19:%.*]] = select i1 [[TMP18]], i64 [[TMP17]], i64 0 -; CHECK-ORDERED-TF-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-TF-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], 32 -; CHECK-ORDERED-TF-NEXT: [[TMP22:%.*]] = sub i64 [[N]], [[TMP21]] -; CHECK-ORDERED-TF-NEXT: [[TMP23:%.*]] = icmp ugt i64 [[N]], [[TMP21]] -; CHECK-ORDERED-TF-NEXT: [[TMP24:%.*]] = select i1 [[TMP23]], i64 [[TMP22]], i64 0 ; CHECK-ORDERED-TF-NEXT: [[TMP25:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-ORDERED-TF-NEXT: [[TMP26:%.*]] = mul i64 [[TMP25]], 8 ; CHECK-ORDERED-TF-NEXT: [[INDEX_PART_NEXT:%.*]] = add i64 0, [[TMP26]] @@ -492,9 +477,9 @@ define float @fadd_strict_unroll(ptr noalias nocapture readonly %a, i64 %n) #0 { ; CHECK-ORDERED-TF-NEXT: [[TMP78:%.*]] = mul i64 [[TMP77]], 24 ; CHECK-ORDERED-TF-NEXT: [[TMP79:%.*]] = add i64 [[INDEX]], [[TMP78]] ; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX]], i64 [[TMP9]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT12]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP73]], i64 [[TMP14]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT13]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP76]], i64 [[TMP19]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT14]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP79]], i64 [[TMP24]]) +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT12]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP73]], i64 [[TMP9]]) +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT13]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP76]], i64 [[TMP9]]) +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT14]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP79]], i64 [[TMP9]]) ; CHECK-ORDERED-TF-NEXT: [[TMP80:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) ; CHECK-ORDERED-TF-NEXT: [[TMP81:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT12]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) ; CHECK-ORDERED-TF-NEXT: [[TMP82:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT13]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -1715,21 +1700,6 @@ define float @fmuladd_strict(ptr %a, ptr %b, i64 %n) #0 { ; CHECK-ORDERED-TF-NEXT: [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]] ; CHECK-ORDERED-TF-NEXT: [[TMP8:%.*]] = icmp ugt i64 [[N]], [[TMP6]] ; CHECK-ORDERED-TF-NEXT: [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0 -; CHECK-ORDERED-TF-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-TF-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 32 -; CHECK-ORDERED-TF-NEXT: [[TMP12:%.*]] = sub i64 [[N]], [[TMP11]] -; CHECK-ORDERED-TF-NEXT: [[TMP13:%.*]] = icmp ugt i64 [[N]], [[TMP11]] -; CHECK-ORDERED-TF-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 0 -; CHECK-ORDERED-TF-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-TF-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 32 -; CHECK-ORDERED-TF-NEXT: [[TMP17:%.*]] = sub i64 [[N]], [[TMP16]] -; CHECK-ORDERED-TF-NEXT: [[TMP18:%.*]] = icmp ugt i64 [[N]], [[TMP16]] -; CHECK-ORDERED-TF-NEXT: [[TMP19:%.*]] = select i1 [[TMP18]], i64 [[TMP17]], i64 0 -; CHECK-ORDERED-TF-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-TF-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], 32 -; CHECK-ORDERED-TF-NEXT: [[TMP22:%.*]] = sub i64 [[N]], [[TMP21]] -; CHECK-ORDERED-TF-NEXT: [[TMP23:%.*]] = icmp ugt i64 [[N]], [[TMP21]] -; CHECK-ORDERED-TF-NEXT: [[TMP24:%.*]] = select i1 [[TMP23]], i64 [[TMP22]], i64 0 ; CHECK-ORDERED-TF-NEXT: [[TMP25:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-ORDERED-TF-NEXT: [[TMP26:%.*]] = mul i64 [[TMP25]], 8 ; CHECK-ORDERED-TF-NEXT: [[INDEX_PART_NEXT:%.*]] = add i64 0, [[TMP26]] @@ -1826,9 +1796,9 @@ define float @fmuladd_strict(ptr %a, ptr %b, i64 %n) #0 { ; CHECK-ORDERED-TF-NEXT: [[TMP96:%.*]] = mul i64 [[TMP95]], 24 ; CHECK-ORDERED-TF-NEXT: [[TMP97:%.*]] = add i64 [[INDEX]], [[TMP96]] ; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX]], i64 [[TMP9]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT16]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP91]], i64 [[TMP14]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT17]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP94]], i64 [[TMP19]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT18]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP97]], i64 [[TMP24]]) +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT16]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP91]], i64 [[TMP9]]) +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT17]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP94]], i64 [[TMP9]]) +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT18]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP97]], i64 [[TMP9]]) ; CHECK-ORDERED-TF-NEXT: [[TMP98:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) ; CHECK-ORDERED-TF-NEXT: [[TMP99:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT16]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) ; CHECK-ORDERED-TF-NEXT: [[TMP100:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT17]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -2129,21 +2099,6 @@ define float @fmuladd_strict_fmf(ptr %a, ptr %b, i64 %n) #0 { ; CHECK-ORDERED-TF-NEXT: [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]] ; CHECK-ORDERED-TF-NEXT: [[TMP8:%.*]] = icmp ugt i64 [[N]], [[TMP6]] ; CHECK-ORDERED-TF-NEXT: [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0 -; CHECK-ORDERED-TF-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-TF-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 32 -; CHECK-ORDERED-TF-NEXT: [[TMP12:%.*]] = sub i64 [[N]], [[TMP11]] -; CHECK-ORDERED-TF-NEXT: [[TMP13:%.*]] = icmp ugt i64 [[N]], [[TMP11]] -; CHECK-ORDERED-TF-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 0 -; CHECK-ORDERED-TF-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-TF-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 32 -; CHECK-ORDERED-TF-NEXT: [[TMP17:%.*]] = sub i64 [[N]], [[TMP16]] -; CHECK-ORDERED-TF-NEXT: [[TMP18:%.*]] = icmp ugt i64 [[N]], [[TMP16]] -; CHECK-ORDERED-TF-NEXT: [[TMP19:%.*]] = select i1 [[TMP18]], i64 [[TMP17]], i64 0 -; CHECK-ORDERED-TF-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-TF-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], 32 -; CHECK-ORDERED-TF-NEXT: [[TMP22:%.*]] = sub i64 [[N]], [[TMP21]] -; CHECK-ORDERED-TF-NEXT: [[TMP23:%.*]] = icmp ugt i64 [[N]], [[TMP21]] -; CHECK-ORDERED-TF-NEXT: [[TMP24:%.*]] = select i1 [[TMP23]], i64 [[TMP22]], i64 0 ; CHECK-ORDERED-TF-NEXT: [[TMP25:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-ORDERED-TF-NEXT: [[TMP26:%.*]] = mul i64 [[TMP25]], 8 ; CHECK-ORDERED-TF-NEXT: [[INDEX_PART_NEXT:%.*]] = add i64 0, [[TMP26]] @@ -2240,9 +2195,9 @@ define float @fmuladd_strict_fmf(ptr %a, ptr %b, i64 %n) #0 { ; CHECK-ORDERED-TF-NEXT: [[TMP96:%.*]] = mul i64 [[TMP95]], 24 ; CHECK-ORDERED-TF-NEXT: [[TMP97:%.*]] = add i64 [[INDEX]], [[TMP96]] ; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX]], i64 [[TMP9]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT16]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP91]], i64 [[TMP14]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT17]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP94]], i64 [[TMP19]]) -; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT18]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP97]], i64 [[TMP24]]) +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT16]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP91]], i64 [[TMP9]]) +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT17]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP94]], i64 [[TMP9]]) +; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT18]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP97]], i64 [[TMP9]]) ; CHECK-ORDERED-TF-NEXT: [[TMP98:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) ; CHECK-ORDERED-TF-NEXT: [[TMP99:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT16]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) ; CHECK-ORDERED-TF-NEXT: [[TMP100:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT17]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll index 1a6e83a61ce7..2acc1ddbffea 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll @@ -25,21 +25,6 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 { ; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]] ; CHECK-NEXT: [[TMP8:%.*]] = icmp ugt i64 [[UMAX]], [[TMP6]] ; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0 -; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 16 -; CHECK-NEXT: [[TMP12:%.*]] = sub i64 [[UMAX]], [[TMP11]] -; CHECK-NEXT: [[TMP13:%.*]] = icmp ugt i64 [[UMAX]], [[TMP11]] -; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 0 -; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 16 -; CHECK-NEXT: [[TMP17:%.*]] = sub i64 [[UMAX]], [[TMP16]] -; CHECK-NEXT: [[TMP18:%.*]] = icmp ugt i64 [[UMAX]], [[TMP16]] -; CHECK-NEXT: [[TMP19:%.*]] = select i1 [[TMP18]], i64 [[TMP17]], i64 0 -; CHECK-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], 16 -; CHECK-NEXT: [[TMP22:%.*]] = sub i64 [[UMAX]], [[TMP21]] -; CHECK-NEXT: [[TMP23:%.*]] = icmp ugt i64 [[UMAX]], [[TMP21]] -; CHECK-NEXT: [[TMP24:%.*]] = select i1 [[TMP23]], i64 [[TMP22]], i64 0 ; CHECK-NEXT: [[TMP25:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP26:%.*]] = mul i64 [[TMP25]], 4 ; CHECK-NEXT: [[INDEX_PART_NEXT:%.*]] = add i64 0, [[TMP26]] @@ -107,9 +92,9 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 { ; CHECK-NEXT: [[TMP70:%.*]] = mul i64 [[TMP69]], 12 ; CHECK-NEXT: [[TMP71:%.*]] = add i64 [[INDEX6]], [[TMP70]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX6]], i64 [[TMP9]]) -; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT11]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP65]], i64 [[TMP14]]) -; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT12]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP68]], i64 [[TMP19]]) -; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT13]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP71]], i64 [[TMP24]]) +; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT11]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP65]], i64 [[TMP9]]) +; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT12]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP68]], i64 [[TMP9]]) +; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT13]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP71]], i64 [[TMP9]]) ; CHECK-NEXT: [[TMP72:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) ; CHECK-NEXT: [[TMP73:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT11]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) ; CHECK-NEXT: [[TMP74:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT12]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -167,21 +152,6 @@ define void @cond_memset(i32 %val, ptr noalias readonly %cond_ptr, ptr noalias % ; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]] ; CHECK-NEXT: [[TMP8:%.*]] = icmp ugt i64 [[UMAX]], [[TMP6]] ; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0 -; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 16 -; CHECK-NEXT: [[TMP12:%.*]] = sub i64 [[UMAX]], [[TMP11]] -; CHECK-NEXT: [[TMP13:%.*]] = icmp ugt i64 [[UMAX]], [[TMP11]] -; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 0 -; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 16 -; CHECK-NEXT: [[TMP17:%.*]] = sub i64 [[UMAX]], [[TMP16]] -; CHECK-NEXT: [[TMP18:%.*]] = icmp ugt i64 [[UMAX]], [[TMP16]] -; CHECK-NEXT: [[TMP19:%.*]] = select i1 [[TMP18]], i64 [[TMP17]], i64 0 -; CHECK-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], 16 -; CHECK-NEXT: [[TMP22:%.*]] = sub i64 [[UMAX]], [[TMP21]] -; CHECK-NEXT: [[TMP23:%.*]] = icmp ugt i64 [[UMAX]], [[TMP21]] -; CHECK-NEXT: [[TMP24:%.*]] = select i1 [[TMP23]], i64 [[TMP22]], i64 0 ; CHECK-NEXT: [[TMP25:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP26:%.*]] = mul i64 [[TMP25]], 4 ; CHECK-NEXT: [[INDEX_PART_NEXT:%.*]] = add i64 0, [[TMP26]] @@ -275,9 +245,9 @@ define void @cond_memset(i32 %val, ptr noalias readonly %cond_ptr, ptr noalias % ; CHECK-NEXT: [[TMP92:%.*]] = mul i64 [[TMP91]], 12 ; CHECK-NEXT: [[TMP93:%.*]] = add i64 [[INDEX6]], [[TMP92]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX6]], i64 [[TMP9]]) -; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT14]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP87]], i64 [[TMP14]]) -; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT15]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP90]], i64 [[TMP19]]) -; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT16]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP93]], i64 [[TMP24]]) +; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT14]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP87]], i64 [[TMP9]]) +; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT15]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP90]], i64 [[TMP9]]) +; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT16]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP93]], i64 [[TMP9]]) ; CHECK-NEXT: [[TMP94:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) ; CHECK-NEXT: [[TMP95:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT14]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) ; CHECK-NEXT: [[TMP96:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT15]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll b/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll index 4957bbeda671..d8f14f30295b 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll @@ -40,9 +40,6 @@ define void @test_uniform(ptr noalias %dst, ptr readonly %src, i64 %uniform , i6 ; INTERLEAVE-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() ; INTERLEAVE-NEXT: [[TMP3:%.*]] = shl i64 [[TMP2]], 2 ; INTERLEAVE-NEXT: [[TMP4:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[N]], i64 [[TMP3]]) -; INTERLEAVE-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() -; INTERLEAVE-NEXT: [[TMP6:%.*]] = shl i64 [[TMP5]], 2 -; INTERLEAVE-NEXT: [[TMP7:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[N]], i64 [[TMP6]]) ; INTERLEAVE-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() ; INTERLEAVE-NEXT: [[TMP9:%.*]] = shl i64 [[TMP8]], 1 ; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 [[N]]) @@ -71,7 +68,7 @@ define void @test_uniform(ptr noalias %dst, ptr readonly %src, i64 %uniform , i6 ; INTERLEAVE-NEXT: [[TMP21:%.*]] = shl i64 [[TMP20]], 1 ; INTERLEAVE-NEXT: [[TMP22:%.*]] = add i64 [[INDEX]], [[TMP21]] ; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX]], i64 [[TMP4]]) -; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT4]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP22]], i64 [[TMP7]]) +; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT4]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP22]], i64 [[TMP4]]) ; INTERLEAVE-NEXT: [[TMP23:%.*]] = extractelement [[ACTIVE_LANE_MASK_NEXT]], i64 0 ; INTERLEAVE-NEXT: br i1 [[TMP23]], label [[VECTOR_BODY]], label [[FOR_COND_CLEANUP:%.*]], !llvm.loop [[LOOP0:![0-9]+]] ; INTERLEAVE: for.cond.cleanup: @@ -129,9 +126,6 @@ define void @test_uniform_smaller_scalar(ptr noalias %dst, ptr readonly %src, i3 ; INTERLEAVE-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() ; INTERLEAVE-NEXT: [[TMP3:%.*]] = shl i64 [[TMP2]], 2 ; INTERLEAVE-NEXT: [[TMP4:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[N]], i64 [[TMP3]]) -; INTERLEAVE-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() -; INTERLEAVE-NEXT: [[TMP6:%.*]] = shl i64 [[TMP5]], 2 -; INTERLEAVE-NEXT: [[TMP7:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[N]], i64 [[TMP6]]) ; INTERLEAVE-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() ; INTERLEAVE-NEXT: [[TMP9:%.*]] = shl i64 [[TMP8]], 1 ; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 [[N]]) @@ -160,7 +154,7 @@ define void @test_uniform_smaller_scalar(ptr noalias %dst, ptr readonly %src, i3 ; INTERLEAVE-NEXT: [[TMP21:%.*]] = shl i64 [[TMP20]], 1 ; INTERLEAVE-NEXT: [[TMP22:%.*]] = add i64 [[INDEX]], [[TMP21]] ; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX]], i64 [[TMP4]]) -; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT4]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP22]], i64 [[TMP7]]) +; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT4]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP22]], i64 [[TMP4]]) ; INTERLEAVE-NEXT: [[TMP23:%.*]] = extractelement [[ACTIVE_LANE_MASK_NEXT]], i64 0 ; INTERLEAVE-NEXT: br i1 [[TMP23]], label [[VECTOR_BODY]], label [[FOR_COND_CLEANUP:%.*]], !llvm.loop [[LOOP3:![0-9]+]] ; INTERLEAVE: for.cond.cleanup: @@ -207,7 +201,6 @@ define void @test_uniform_not_invariant(ptr noalias %dst, ptr readonly %src, i64 ; INTERLEAVE-SAME: (ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { ; INTERLEAVE-NEXT: entry: ; INTERLEAVE-NEXT: [[TMP0:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[N]], i64 2) -; INTERLEAVE-NEXT: [[TMP1:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[N]], i64 2) ; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = icmp ne i64 [[N]], 0 ; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = icmp ugt i64 [[N]], 1 ; INTERLEAVE-NEXT: br label [[VECTOR_BODY:%.*]] @@ -237,7 +230,7 @@ define void @test_uniform_not_invariant(ptr noalias %dst, ptr readonly %src, i64 ; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 2 ; INTERLEAVE-NEXT: [[TMP11:%.*]] = or disjoint i64 [[INDEX]], 1 ; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = icmp ult i64 [[INDEX]], [[TMP0]] -; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT5]] = icmp ult i64 [[TMP11]], [[TMP1]] +; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT5]] = icmp ult i64 [[TMP11]], [[TMP0]] ; INTERLEAVE-NEXT: br i1 [[ACTIVE_LANE_MASK_NEXT]], label [[VECTOR_BODY]], label [[FOR_COND_CLEANUP:%.*]], !llvm.loop [[LOOP4:![0-9]+]] ; INTERLEAVE: for.cond.cleanup: ; INTERLEAVE-NEXT: ret void -- GitLab From 18a49f03aa2b6bfeb073648b9eb75277a2386fc4 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Sun, 24 Mar 2024 14:07:09 -0700 Subject: [PATCH 175/306] [ELF] Merge relaIplt into relaDyn `relaIplt` was added so that IRELATIVE relocations are placed at the end of .rela.dyn (since https://reviews.llvm.org/D65651) or .rela.plt (--pack-dyn-relocs=android[+relr]). Unfortunately, handling `relaIplt` requires special cases all over the code base. We can extend partitionRels/computeRels to partition both RELATIVE and IRELATIVE relocations, rendering `relaIplt` unneeded. The change allows IRELATIVE relocations in the DT_ANDROID_REL[A] table (untested?!), which may be processed before other types of relocations. This seems acceptable for Bionic's DEFINE_IFUNC_FOR use cases. In addition, this change simplies changing .rel[a].dyn to a compact relocation format (CREL). SHF_INFO_LINK is removed from .rel[a].dyn with IRELATIVE relocations. (See https://reviews.llvm.org/D89828). --- lld/ELF/Relocations.cpp | 10 ++--- lld/ELF/SyntheticSections.cpp | 19 +++----- lld/ELF/SyntheticSections.h | 1 - lld/ELF/Writer.cpp | 50 ++++++--------------- lld/test/ELF/aarch64-gnu-ifunc.s | 3 +- lld/test/ELF/arm-gnu-ifunc.s | 3 +- lld/test/ELF/gnu-ifunc-i386.s | 3 +- lld/test/ELF/systemz-ifunc-nonpreemptible.s | 2 +- 8 files changed, 28 insertions(+), 63 deletions(-) diff --git a/lld/ELF/Relocations.cpp b/lld/ELF/Relocations.cpp index 92a1b9baaca3..33c50133bec4 100644 --- a/lld/ELF/Relocations.cpp +++ b/lld/ELF/Relocations.cpp @@ -1620,12 +1620,8 @@ static bool handleNonPreemptibleIfunc(Symbol &sym, uint16_t flags) { // relatively straightforward. We create a PLT entry in Iplt, which is // usually at the end of .plt, which makes an indirect call using a // matching GOT entry in igotPlt, which is usually at the end of .got.plt. - // The GOT entry is relocated using an IRELATIVE relocation in relaIplt, - // which is usually at the end of .rela.plt. Unlike most relocations in - // .rela.plt, which may be evaluated lazily without -z now, dynamic - // loaders evaluate IRELATIVE relocs eagerly, which means that for - // IRELATIVE relocs only, GOT-generating relocations can point directly to - // .got.plt without requiring a separate GOT entry. + // The GOT entry is relocated using an IRELATIVE relocation in relaDyn, + // which is usually at the end of .rela.dyn. // // - Despite the fact that an ifunc does not have a fixed value, compilers // that are not passed -fPIC will assume that they do, and will emit @@ -1665,7 +1661,7 @@ static bool handleNonPreemptibleIfunc(Symbol &sym, uint16_t flags) { // section/value fixed. auto *directSym = makeDefined(cast(sym)); directSym->allocateAux(); - addPltEntry(*in.iplt, *in.igotPlt, *in.relaIplt, target->iRelativeRel, + addPltEntry(*in.iplt, *in.igotPlt, *mainPart->relaDyn, target->iRelativeRel, *directSym); sym.allocateAux(); symAux.back().pltIdx = symAux[directSym->auxIdx].pltIdx; diff --git a/lld/ELF/SyntheticSections.cpp b/lld/ELF/SyntheticSections.cpp index 10eda17f0cb3..f924756ddddf 100644 --- a/lld/ELF/SyntheticSections.cpp +++ b/lld/ELF/SyntheticSections.cpp @@ -1267,15 +1267,12 @@ DynamicSection::DynamicSection() // The output section .rela.dyn may include these synthetic sections: // // - part.relaDyn -// - in.relaIplt: this is included if in.relaIplt is named .rela.dyn // - in.relaPlt: this is included if a linker script places .rela.plt inside // .rela.dyn // // DT_RELASZ is the total size of the included sections. static uint64_t addRelaSz(const RelocationBaseSection &relaDyn) { size_t size = relaDyn.getSize(); - if (in.relaIplt->getParent() == relaDyn.getParent()) - size += in.relaIplt->getSize(); if (in.relaPlt->getParent() == relaDyn.getParent()) size += in.relaPlt->getSize(); return size; @@ -1372,9 +1369,7 @@ DynamicSection::computeContents() { if (!config->shared && !config->relocatable && !config->zRodynamic) addInt(DT_DEBUG, 0); - if (part.relaDyn->isNeeded() || - (in.relaIplt->isNeeded() && - part.relaDyn->getParent() == in.relaIplt->getParent())) { + if (part.relaDyn->isNeeded()) { addInSec(part.relaDyn->dynamicTag, *part.relaDyn); entries.emplace_back(part.relaDyn->sizeDynamicTag, addRelaSz(*part.relaDyn)); @@ -1657,10 +1652,6 @@ void RelocationBaseSection::finalizeContents() { getParent()->flags |= ELF::SHF_INFO_LINK; getParent()->info = in.gotPlt->getParent()->sectionIndex; } - if (in.relaIplt.get() == this && in.igotPlt->getParent()) { - getParent()->flags |= ELF::SHF_INFO_LINK; - getParent()->info = in.igotPlt->getParent()->sectionIndex; - } } void DynamicReloc::computeRaw(SymbolTableBaseSection *symtab) { @@ -1674,6 +1665,11 @@ void RelocationBaseSection::computeRels() { SymbolTableBaseSection *symTab = getPartition().dynSymTab.get(); parallelForEach(relocs, [symTab](DynamicReloc &rel) { rel.computeRaw(symTab); }); + + auto irelative = std::partition( + relocs.begin() + numRelativeRelocs, relocs.end(), + [t = target->iRelativeRel](auto &r) { return r.type != t; }); + // Sort by (!IsRelative,SymIndex,r_offset). DT_REL[A]COUNT requires us to // place R_*_RELATIVE first. SymIndex is to improve locality, while r_offset // is to make results easier to read. @@ -1682,7 +1678,7 @@ void RelocationBaseSection::computeRels() { parallelSort(relocs.begin(), nonRelative, [&](auto &a, auto &b) { return a.r_offset < b.r_offset; }); // Non-relative relocations are few, so don't bother with parallelSort. - llvm::sort(nonRelative, relocs.end(), [&](auto &a, auto &b) { + llvm::sort(nonRelative, irelative, [&](auto &a, auto &b) { return std::tie(a.r_sym, a.r_offset) < std::tie(b.r_sym, b.r_offset); }); } @@ -3843,7 +3839,6 @@ void InStruct::reset() { ppc32Got2.reset(); ibtPlt.reset(); relaPlt.reset(); - relaIplt.reset(); shStrTab.reset(); strTab.reset(); symTab.reset(); diff --git a/lld/ELF/SyntheticSections.h b/lld/ELF/SyntheticSections.h index b41e69410054..fa21b80a5a5e 100644 --- a/lld/ELF/SyntheticSections.h +++ b/lld/ELF/SyntheticSections.h @@ -1358,7 +1358,6 @@ struct InStruct { std::unique_ptr ppc32Got2; std::unique_ptr ibtPlt; std::unique_ptr relaPlt; - std::unique_ptr relaIplt; std::unique_ptr shStrTab; std::unique_ptr strTab; std::unique_ptr symTab; diff --git a/lld/ELF/Writer.cpp b/lld/ELF/Writer.cpp index 4eca7b22e90b..40d617b7fdf3 100644 --- a/lld/ELF/Writer.cpp +++ b/lld/ELF/Writer.cpp @@ -449,8 +449,8 @@ template void elf::createSyntheticSections() { add(*part.dynamic); add(*part.dynStrTab); - add(*part.relaDyn); } + add(*part.relaDyn); if (config->relrPackDynRelocs) { part.relrDyn = std::make_unique>(threadCount); @@ -550,17 +550,6 @@ template void elf::createSyntheticSections() { /*threadCount=*/1); add(*in.relaPlt); - // The relaIplt immediately follows .rel[a].dyn to ensure that the IRelative - // relocations are processed last by the dynamic loader. We cannot place the - // iplt section in .rel.dyn when Android relocation packing is enabled because - // that would cause a section type mismatch. However, because the Android - // dynamic loader reads .rel.plt after .rel.dyn, we can get the desired - // behaviour by placing the iplt section in .rel.plt. - in.relaIplt = std::make_unique>( - config->androidPackDynRelocs ? in.relaPlt->name : relaDynName, - /*sort=*/false, /*threadCount=*/1); - add(*in.relaIplt); - if ((config->emachine == EM_386 || config->emachine == EM_X86_64) && (config->andFeatures & GNU_PROPERTY_X86_FEATURE_1_IBT)) { in.ibtPlt = std::make_unique(); @@ -1071,20 +1060,18 @@ void PhdrEntry::add(OutputSection *sec) { sec->ptLoad = this; } -// The beginning and the ending of .rel[a].plt section are marked -// with __rel[a]_iplt_{start,end} symbols if it is a statically linked -// executable. The runtime needs these symbols in order to resolve -// all IRELATIVE relocs on startup. For dynamic executables, we don't -// need these symbols, since IRELATIVE relocs are resolved through GOT -// and PLT. For details, see http://www.airs.com/blog/archives/403. +// A statically linked position-dependent executable should only contain +// IRELATIVE relocations and no other dynamic relocations. Encapsulation symbols +// __rel[a]_iplt_{start,end} will be defined for .rel[a].dyn, to be +// processed by the libc runtime. Other executables or DSOs use dynamic tags +// instead. template void Writer::addRelIpltSymbols() { if (config->isPic) return; - // By default, __rela_iplt_{start,end} belong to a dummy section 0 - // because .rela.plt might be empty and thus removed from output. - // We'll override Out::elfHeader with In.relaIplt later when we are - // sure that .rela.plt exists in output. + // __rela_iplt_{start,end} are initially defined relative to dummy section 0. + // We'll override Out::elfHeader with relaDyn later when we are sure that + // .rela.dyn will be present in the output. ElfSym::relaIpltStart = addOptionalRegular( config->isRela ? "__rela_iplt_start" : "__rel_iplt_start", Out::elfHeader, 0, STV_HIDDEN); @@ -1110,11 +1097,11 @@ template void Writer::setReservedSymbolSections() { ElfSym::globalOffsetTable->section = sec; } - // .rela_iplt_{start,end} mark the start and the end of in.relaIplt. - if (ElfSym::relaIpltStart && in.relaIplt->isNeeded()) { - ElfSym::relaIpltStart->section = in.relaIplt.get(); - ElfSym::relaIpltEnd->section = in.relaIplt.get(); - ElfSym::relaIpltEnd->value = in.relaIplt->getSize(); + // .rela_iplt_{start,end} mark the start and the end of .rel[a].dyn. + if (ElfSym::relaIpltStart && mainPart->relaDyn->isNeeded()) { + ElfSym::relaIpltStart->section = mainPart->relaDyn.get(); + ElfSym::relaIpltEnd->section = mainPart->relaDyn.get(); + ElfSym::relaIpltEnd->value = mainPart->relaDyn->getSize(); } PhdrEntry *last = nullptr; @@ -1470,14 +1457,6 @@ static void sortSection(OutputSection &osec, if (name == ".init" || name == ".fini") return; - // IRelative relocations that usually live in the .rel[a].dyn section should - // be processed last by the dynamic loader. To achieve that we add synthetic - // sections in the required order from the beginning so that the in.relaIplt - // section is placed last in an output section. Here we just do not apply - // sorting for an output section which holds the in.relaIplt section. - if (in.relaIplt->getParent() == &osec) - return; - // Sort input sections by priority using the list provided by // --symbol-ordering-file or --shuffle-sections=. This is a least significant // digit radix sort. The sections may be sorted stably again by a more @@ -2196,7 +2175,6 @@ template void Writer::finalizeSections() { finalizeSynthetic(in.mipsGot.get()); finalizeSynthetic(in.igotPlt.get()); finalizeSynthetic(in.gotPlt.get()); - finalizeSynthetic(in.relaIplt.get()); finalizeSynthetic(in.relaPlt.get()); finalizeSynthetic(in.plt.get()); finalizeSynthetic(in.iplt.get()); diff --git a/lld/test/ELF/aarch64-gnu-ifunc.s b/lld/test/ELF/aarch64-gnu-ifunc.s index dee24779d913..d76b54eabf8a 100644 --- a/lld/test/ELF/aarch64-gnu-ifunc.s +++ b/lld/test/ELF/aarch64-gnu-ifunc.s @@ -11,13 +11,12 @@ // CHECK-NEXT: Type: SHT_RELA // CHECK-NEXT: Flags [ // CHECK-NEXT: SHF_ALLOC -// CHECK-NEXT: SHF_INFO_LINK // CHECK-NEXT: ] // CHECK-NEXT: Address: [[RELA:.*]] // CHECK-NEXT: Offset: 0x158 // CHECK-NEXT: Size: 48 // CHECK-NEXT: Link: 0 -// CHECK-NEXT: Info: 4 +// CHECK-NEXT: Info: 0 // CHECK-NEXT: AddressAlignment: 8 // CHECK-NEXT: EntrySize: 24 // CHECK-NEXT: } diff --git a/lld/test/ELF/arm-gnu-ifunc.s b/lld/test/ELF/arm-gnu-ifunc.s index 562478256fd8..d49ca18e991e 100644 --- a/lld/test/ELF/arm-gnu-ifunc.s +++ b/lld/test/ELF/arm-gnu-ifunc.s @@ -30,13 +30,12 @@ _start: // CHECK-NEXT: Type: SHT_REL // CHECK-NEXT: Flags [ // CHECK-NEXT: SHF_ALLOC -// CHECK-NEXT: SHF_INFO_LINK // CHECK-NEXT: ] // CHECK-NEXT: Address: 0x100F4 // CHECK-NEXT: Offset: 0xF4 // CHECK-NEXT: Size: 16 // CHECK-NEXT: Link: -// CHECK-NEXT: Info: 4 +// CHECK-NEXT: Info: 0 // CHECK: Name: .iplt // CHECK-NEXT: Type: SHT_PROGBITS // CHECK-NEXT: Flags [ diff --git a/lld/test/ELF/gnu-ifunc-i386.s b/lld/test/ELF/gnu-ifunc-i386.s index b502fd6e9ae2..43b19b27ea4e 100644 --- a/lld/test/ELF/gnu-ifunc-i386.s +++ b/lld/test/ELF/gnu-ifunc-i386.s @@ -11,13 +11,12 @@ // CHECK-NEXT: Type: SHT_REL // CHECK-NEXT: Flags [ // CHECK-NEXT: SHF_ALLOC -// CHECK-NEXT: SHF_INFO_LINK // CHECK-NEXT: ] // CHECK-NEXT: Address: [[RELA:.*]] // CHECK-NEXT: Offset: 0xD4 // CHECK-NEXT: Size: 16 // CHECK-NEXT: Link: 0 -// CHECK-NEXT: Info: 4 +// CHECK-NEXT: Info: 0 // CHECK-NEXT: AddressAlignment: 4 // CHECK-NEXT: EntrySize: 8 // CHECK-NEXT: } diff --git a/lld/test/ELF/systemz-ifunc-nonpreemptible.s b/lld/test/ELF/systemz-ifunc-nonpreemptible.s index 5056db302ca1..892bbde8d9c7 100644 --- a/lld/test/ELF/systemz-ifunc-nonpreemptible.s +++ b/lld/test/ELF/systemz-ifunc-nonpreemptible.s @@ -10,7 +10,7 @@ # CHECK: Section Headers: # CHECK-NEXT: [Nr] Name Type Address Off Size ES Flg Lk Inf Al # CHECK-NEXT: [ 0] NULL 0000000000000000 000000 000000 00 0 0 0 -# CHECK-NEXT: [ 1] .rela.dyn RELA 0000000001000158 000158 000030 18 AI 0 4 8 +# CHECK-NEXT: [ 1] .rela.dyn RELA 0000000001000158 000158 000030 18 A 0 0 8 # CHECK-NEXT: [ 2] .text PROGBITS 0000000001001188 000188 00001c 00 AX 0 0 4 # CHECK-NEXT: [ 3] .iplt PROGBITS 00000000010011b0 0001b0 000040 00 AX 0 0 16 # CHECK-NEXT: [ 4] .got.plt PROGBITS 00000000010021f0 0001f0 000010 00 WA 0 0 8 -- GitLab From 67f2267ae006f35edfdf36567646403e61527d1b Mon Sep 17 00:00:00 2001 From: Min Hsu Date: Sun, 24 Mar 2024 14:24:44 -0700 Subject: [PATCH 176/306] [M68k][NFC] Suppress warning on an unused variable when assertion is disabled NFC. --- llvm/lib/Target/M68k/M68kISelLowering.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/lib/Target/M68k/M68kISelLowering.cpp b/llvm/lib/Target/M68k/M68kISelLowering.cpp index 158393f02a24..786aa7bcb64e 100644 --- a/llvm/lib/Target/M68k/M68kISelLowering.cpp +++ b/llvm/lib/Target/M68k/M68kISelLowering.cpp @@ -939,6 +939,7 @@ SDValue M68kTargetLowering::LowerFormalArguments( for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { CCValAssign &VA = ArgLocs[i]; assert(VA.getValNo() != LastVal && "Same value in different locations"); + (void)LastVal; LastVal = VA.getValNo(); -- GitLab From cceedc939a43c7c732a5888364251775bffc2dba Mon Sep 17 00:00:00 2001 From: Owen Pan Date: Sun, 24 Mar 2024 15:22:40 -0700 Subject: [PATCH 177/306] [clang-format] Fix a crash with AlignArrayOfStructures option (#86420) Fixes #86109. --- clang/lib/Format/WhitespaceManager.cpp | 2 +- clang/unittests/Format/FormatTest.cpp | 18 ++++++++++++++++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/clang/lib/Format/WhitespaceManager.cpp b/clang/lib/Format/WhitespaceManager.cpp index fef85abf79a3..710bf8d8a8ec 100644 --- a/clang/lib/Format/WhitespaceManager.cpp +++ b/clang/lib/Format/WhitespaceManager.cpp @@ -1491,7 +1491,7 @@ WhitespaceManager::CellDescriptions WhitespaceManager::getCells(unsigned Start, : Cell); // Go to the next non-comment and ensure there is a break in front const auto *NextNonComment = C.Tok->getNextNonComment(); - while (NextNonComment->is(tok::comma)) + while (NextNonComment && NextNonComment->is(tok::comma)) NextNonComment = NextNonComment->getNextNonComment(); auto j = i; while (j < End && Changes[j].Tok != NextNonComment) diff --git a/clang/unittests/Format/FormatTest.cpp b/clang/unittests/Format/FormatTest.cpp index cf8d6ab691d9..03005384a6f6 100644 --- a/clang/unittests/Format/FormatTest.cpp +++ b/clang/unittests/Format/FormatTest.cpp @@ -21100,7 +21100,14 @@ TEST_F(FormatTest, CatchAlignArrayOfStructuresRightAlignment) { " [0] = {1, 1},\n" " [1] { 1, 1, },\n" " [2] { 1, 1, },\n" - "};"); + "};", + Style); + verifyNoCrash("test arr[] = {\n" + "#define FOO(i) {i, i},\n" + "SOME_GENERATOR(FOO)\n" + "{2, 2}\n" + "};", + Style); verifyFormat("return GradForUnaryCwise(g, {\n" " {{\"sign\"}, \"Sign\", " @@ -21353,7 +21360,14 @@ TEST_F(FormatTest, CatchAlignArrayOfStructuresLeftAlignment) { " [0] = {1, 1},\n" " [1] { 1, 1, },\n" " [2] { 1, 1, },\n" - "};"); + "};", + Style); + verifyNoCrash("test arr[] = {\n" + "#define FOO(i) {i, i},\n" + "SOME_GENERATOR(FOO)\n" + "{2, 2}\n" + "};", + Style); verifyFormat("return GradForUnaryCwise(g, {\n" " {{\"sign\"}, \"Sign\", {\"x\", " -- GitLab From 230b1895c493c511c11541af3b5bc819887c82a8 Mon Sep 17 00:00:00 2001 From: Finn Plummer <50529406+inbelic@users.noreply.github.com> Date: Sun, 24 Mar 2024 18:56:47 -0700 Subject: [PATCH 178/306] [mlir][spirv] Add folding for [S|U|LessThan[Equal] (#85435) Add missing constant propogation folder for [S|U]LessThan[Equal]. Implement additional folding when the operands are equal for all ops. Allows for constant folding in the IndexToSPIRV pass. Part of work #70704 --- .../mlir/Dialect/SPIRV/IR/SPIRVLogicalOps.td | 8 + .../SPIRV/IR/SPIRVCanonicalization.cpp | 82 ++++++++ .../SPIRV/Transforms/canonicalize.mlir | 176 ++++++++++++++++++ 3 files changed, 266 insertions(+) diff --git a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVLogicalOps.td b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVLogicalOps.td index 3ee239d6e1e3..14d639bc26f2 100644 --- a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVLogicalOps.td +++ b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVLogicalOps.td @@ -716,6 +716,8 @@ def SPIRV_SLessThanOp : SPIRV_LogicalBinaryOp<"SLessThan", ``` }]; + + let hasFolder = 1; } // ----- @@ -745,6 +747,8 @@ def SPIRV_SLessThanEqualOp : SPIRV_LogicalBinaryOp<"SLessThanEqual", %5 = spirv.SLessThanEqual %2, %3 : vector<4xi32> ``` }]; + + let hasFolder = 1; } // ----- @@ -886,6 +890,8 @@ def SPIRV_ULessThanOp : SPIRV_LogicalBinaryOp<"ULessThan", %5 = spirv.ULessThan %2, %3 : vector<4xi32> ``` }]; + + let hasFolder = 1; } // ----- @@ -949,6 +955,8 @@ def SPIRV_ULessThanEqualOp : SPIRV_LogicalBinaryOp<"ULessThanEqual", %5 = spirv.ULessThanEqual %2, %3 : vector<4xi32> ``` }]; + + let hasFolder = 1; } #endif // MLIR_DIALECT_SPIRV_IR_LOGICAL_OPS diff --git a/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp b/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp index ff4bace9a4d8..3f1a7826f8b6 100644 --- a/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp +++ b/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp @@ -880,6 +880,88 @@ OpFoldResult spirv::INotEqualOp::fold(spirv::INotEqualOp::FoldAdaptor adaptor) { }); } +//===----------------------------------------------------------------------===// +// spirv.SLessThan +//===----------------------------------------------------------------------===// + +OpFoldResult spirv::SLessThanOp::fold(spirv::SLessThanOp::FoldAdaptor adaptor) { + // x == x -> false + if (getOperand1() == getOperand2()) { + auto falseAttr = BoolAttr::get(getContext(), false); + if (isa(getType())) + return falseAttr; + if (auto vecTy = dyn_cast(getType())) + return SplatElementsAttr::get(vecTy, falseAttr); + } + + return constFoldBinaryOp( + adaptor.getOperands(), getType(), [](const APInt &a, const APInt &b) { + return a.slt(b) ? APInt::getAllOnes(1) : APInt::getZero(1); + }); +} + +//===----------------------------------------------------------------------===// +// spirv.SLessThanEqual +//===----------------------------------------------------------------------===// + +OpFoldResult +spirv::SLessThanEqualOp::fold(spirv::SLessThanEqualOp::FoldAdaptor adaptor) { + // x == x -> true + if (getOperand1() == getOperand2()) { + auto trueAttr = BoolAttr::get(getContext(), true); + if (isa(getType())) + return trueAttr; + if (auto vecTy = dyn_cast(getType())) + return SplatElementsAttr::get(vecTy, trueAttr); + } + + return constFoldBinaryOp( + adaptor.getOperands(), getType(), [](const APInt &a, const APInt &b) { + return a.sle(b) ? APInt::getAllOnes(1) : APInt::getZero(1); + }); +} + +//===----------------------------------------------------------------------===// +// spirv.ULessThan +//===----------------------------------------------------------------------===// + +OpFoldResult spirv::ULessThanOp::fold(spirv::ULessThanOp::FoldAdaptor adaptor) { + // x == x -> false + if (getOperand1() == getOperand2()) { + auto falseAttr = BoolAttr::get(getContext(), false); + if (isa(getType())) + return falseAttr; + if (auto vecTy = dyn_cast(getType())) + return SplatElementsAttr::get(vecTy, falseAttr); + } + + return constFoldBinaryOp( + adaptor.getOperands(), getType(), [](const APInt &a, const APInt &b) { + return a.ult(b) ? APInt::getAllOnes(1) : APInt::getZero(1); + }); +} + +//===----------------------------------------------------------------------===// +// spirv.ULessThanEqual +//===----------------------------------------------------------------------===// + +OpFoldResult +spirv::ULessThanEqualOp::fold(spirv::ULessThanEqualOp::FoldAdaptor adaptor) { + // x == x -> true + if (getOperand1() == getOperand2()) { + auto trueAttr = BoolAttr::get(getContext(), true); + if (isa(getType())) + return trueAttr; + if (auto vecTy = dyn_cast(getType())) + return SplatElementsAttr::get(vecTy, trueAttr); + } + + return constFoldBinaryOp( + adaptor.getOperands(), getType(), [](const APInt &a, const APInt &b) { + return a.ule(b) ? APInt::getAllOnes(1) : APInt::getZero(1); + }); +} + //===----------------------------------------------------------------------===// // spirv.ShiftLeftLogical //===----------------------------------------------------------------------===// diff --git a/mlir/test/Dialect/SPIRV/Transforms/canonicalize.mlir b/mlir/test/Dialect/SPIRV/Transforms/canonicalize.mlir index de21d114e9fc..ed0bd070c43c 100644 --- a/mlir/test/Dialect/SPIRV/Transforms/canonicalize.mlir +++ b/mlir/test/Dialect/SPIRV/Transforms/canonicalize.mlir @@ -1478,6 +1478,182 @@ func.func @const_fold_vector_inotequal() -> vector<3xi1> { // ----- +//===----------------------------------------------------------------------===// +// spirv.SLessThan +//===----------------------------------------------------------------------===// + +// CHECK-LABEL: @slt_same +func.func @slt_same(%arg0 : i32, %arg1 : vector<3xi32>) -> (i1, vector<3xi1>) { + // CHECK-DAG: %[[CFALSE:.*]] = spirv.Constant false + // CHECK-DAG: %[[CVFALSE:.*]] = spirv.Constant dense + %0 = spirv.SLessThan %arg0, %arg0 : i32 + %1 = spirv.SLessThan %arg1, %arg1 : vector<3xi32> + + // CHECK: return %[[CFALSE]], %[[CVFALSE]] + return %0, %1 : i1, vector<3xi1> +} + +// CHECK-LABEL: @const_fold_scalar_slt +func.func @const_fold_scalar_slt() -> (i1, i1) { + %c4 = spirv.Constant 4 : i32 + %c5 = spirv.Constant 5 : i32 + %c6 = spirv.Constant 6 : i32 + + // CHECK-DAG: %[[CTRUE:.*]] = spirv.Constant true + // CHECK-DAG: %[[CFALSE:.*]] = spirv.Constant false + %0 = spirv.SLessThan %c5, %c6 : i32 + %1 = spirv.SLessThan %c5, %c4 : i32 + + // CHECK: return %[[CTRUE]], %[[CFALSE]] + return %0, %1 : i1, i1 +} + +// CHECK-LABEL: @const_fold_vector_slt +func.func @const_fold_vector_slt() -> vector<3xi1> { + %cv0 = spirv.Constant dense<[-1, -4, 3]> : vector<3xi32> + %cv1 = spirv.Constant dense<[-1, -3, 2]> : vector<3xi32> + + // CHECK: %[[RET:.*]] = spirv.Constant dense<[false, true, false]> + %0 = spirv.SLessThan %cv0, %cv1 : vector<3xi32> + + // CHECK: return %[[RET]] + return %0 : vector<3xi1> +} + +// ----- + +//===----------------------------------------------------------------------===// +// spirv.SLessThanEqual +//===----------------------------------------------------------------------===// + +// CHECK-LABEL: @sle_same +func.func @sle_same(%arg0 : i32, %arg1 : vector<3xi32>) -> (i1, vector<3xi1>) { + // CHECK-DAG: %[[CTRUE:.*]] = spirv.Constant true + // CHECK-DAG: %[[CVTRUE:.*]] = spirv.Constant dense + %0 = spirv.SLessThanEqual %arg0, %arg0 : i32 + %1 = spirv.SLessThanEqual %arg1, %arg1 : vector<3xi32> + + // CHECK: return %[[CTRUE]], %[[CVTRUE]] + return %0, %1 : i1, vector<3xi1> +} + +// CHECK-LABEL: @const_fold_scalar_sle +func.func @const_fold_scalar_sle() -> (i1, i1) { + %c4 = spirv.Constant 4 : i32 + %c5 = spirv.Constant 5 : i32 + %c6 = spirv.Constant 6 : i32 + + // CHECK-DAG: %[[CTRUE:.*]] = spirv.Constant true + // CHECK-DAG: %[[CFALSE:.*]] = spirv.Constant false + %0 = spirv.SLessThanEqual %c5, %c6 : i32 + %1 = spirv.SLessThanEqual %c5, %c4 : i32 + + // CHECK: return %[[CTRUE]], %[[CFALSE]] + return %0, %1 : i1, i1 +} + +// CHECK-LABEL: @const_fold_vector_sle +func.func @const_fold_vector_sle() -> vector<3xi1> { + %cv0 = spirv.Constant dense<[-1, -4, 3]> : vector<3xi32> + %cv1 = spirv.Constant dense<[-1, -3, 2]> : vector<3xi32> + + // CHECK: %[[RET:.*]] = spirv.Constant dense<[true, true, false]> + %0 = spirv.SLessThanEqual %cv0, %cv1 : vector<3xi32> + + // CHECK: return %[[RET]] + return %0 : vector<3xi1> +} + +// ----- + +//===----------------------------------------------------------------------===// +// spirv.ULessThan +//===----------------------------------------------------------------------===// + +// CHECK-LABEL: @ult_same +func.func @ult_same(%arg0 : i32, %arg1 : vector<3xi32>) -> (i1, vector<3xi1>) { + // CHECK-DAG: %[[CFALSE:.*]] = spirv.Constant false + // CHECK-DAG: %[[CVFALSE:.*]] = spirv.Constant dense + %0 = spirv.ULessThan %arg0, %arg0 : i32 + %1 = spirv.ULessThan %arg1, %arg1 : vector<3xi32> + + // CHECK: return %[[CFALSE]], %[[CVFALSE]] + return %0, %1 : i1, vector<3xi1> +} + +// CHECK-LABEL: @const_fold_scalar_ult +func.func @const_fold_scalar_ult() -> (i1, i1) { + %c4 = spirv.Constant 4 : i32 + %c5 = spirv.Constant 5 : i32 + %cn6 = spirv.Constant -6 : i32 + + // CHECK-DAG: %[[CTRUE:.*]] = spirv.Constant true + // CHECK-DAG: %[[CFALSE:.*]] = spirv.Constant false + %0 = spirv.ULessThan %c5, %cn6 : i32 + %1 = spirv.ULessThan %c5, %c4 : i32 + + // CHECK: return %[[CTRUE]], %[[CFALSE]] + return %0, %1 : i1, i1 +} + +// CHECK-LABEL: @const_fold_vector_ult +func.func @const_fold_vector_ult() -> vector<3xi1> { + %cv0 = spirv.Constant dense<[-1, -4, 3]> : vector<3xi32> + %cv1 = spirv.Constant dense<[-1, -3, 2]> : vector<3xi32> + + // CHECK: %[[RET:.*]] = spirv.Constant dense<[false, true, false]> + %0 = spirv.ULessThan %cv0, %cv1 : vector<3xi32> + + // CHECK: return %[[RET]] + return %0 : vector<3xi1> +} + +// ----- + +//===----------------------------------------------------------------------===// +// spirv.ULessThanEqual +//===----------------------------------------------------------------------===// + +// CHECK-LABEL: @ule_same +func.func @ule_same(%arg0 : i32, %arg1 : vector<3xi32>) -> (i1, vector<3xi1>) { + // CHECK-DAG: %[[CTRUE:.*]] = spirv.Constant true + // CHECK-DAG: %[[CVTRUE:.*]] = spirv.Constant dense + %0 = spirv.ULessThanEqual %arg0, %arg0 : i32 + %1 = spirv.ULessThanEqual %arg1, %arg1 : vector<3xi32> + + // CHECK: return %[[CTRUE]], %[[CVTRUE]] + return %0, %1 : i1, vector<3xi1> +} + +// CHECK-LABEL: @const_fold_scalar_ule +func.func @const_fold_scalar_ule() -> (i1, i1) { + %c4 = spirv.Constant 4 : i32 + %c5 = spirv.Constant 5 : i32 + %cn6 = spirv.Constant -6 : i32 + + // CHECK-DAG: %[[CTRUE:.*]] = spirv.Constant true + // CHECK-DAG: %[[CFALSE:.*]] = spirv.Constant false + %0 = spirv.ULessThanEqual %c5, %cn6 : i32 + %1 = spirv.ULessThanEqual %c5, %c4 : i32 + + // CHECK: return %[[CTRUE]], %[[CFALSE]] + return %0, %1 : i1, i1 +} + +// CHECK-LABEL: @const_fold_vector_ule +func.func @const_fold_vector_ule() -> vector<3xi1> { + %cv0 = spirv.Constant dense<[-1, -4, 3]> : vector<3xi32> + %cv1 = spirv.Constant dense<[-1, -3, 2]> : vector<3xi32> + + // CHECK: %[[RET:.*]] = spirv.Constant dense<[true, true, false]> + %0 = spirv.ULessThanEqual %cv0, %cv1 : vector<3xi32> + + // CHECK: return %[[RET]] + return %0 : vector<3xi1> +} + +// ----- + //===----------------------------------------------------------------------===// // spirv.LeftShiftLogical //===----------------------------------------------------------------------===// -- GitLab From 7d2d8e2a7245e4e64da22cb3c422ea3be5a0bf0a Mon Sep 17 00:00:00 2001 From: Kai Sasaki Date: Mon, 25 Mar 2024 10:59:42 +0900 Subject: [PATCH 179/306] [mlir][complex] Fastmath flag for the trigonometric ops in complex (#85563) Support Fastmath flag to convert trigonometric ops in the complex dialect. See: https://discourse.llvm.org/t/rfc-fastmath-flags-support-in-complex-dialect/71981 --- .../ComplexToStandard/ComplexToStandard.cpp | 50 +++++++++++-------- .../convert-to-standard.mlir | 46 +++++++++++++++++ 2 files changed, 75 insertions(+), 21 deletions(-) diff --git a/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp b/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp index 76729278ec1b..17f64f1b65b7 100644 --- a/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp +++ b/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp @@ -196,6 +196,7 @@ struct TrigonometricOpConversion : public OpConversionPattern { auto loc = op.getLoc(); auto type = cast(adaptor.getComplex().getType()); auto elementType = cast(type.getElementType()); + arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr(); Value real = rewriter.create(loc, elementType, adaptor.getComplex()); @@ -207,14 +208,14 @@ struct TrigonometricOpConversion : public OpConversionPattern { // implementation in the subclass to combine them. Value half = rewriter.create( loc, elementType, rewriter.getFloatAttr(elementType, 0.5)); - Value exp = rewriter.create(loc, imag); - Value scaledExp = rewriter.create(loc, half, exp); - Value reciprocalExp = rewriter.create(loc, half, exp); - Value sin = rewriter.create(loc, real); - Value cos = rewriter.create(loc, real); + Value exp = rewriter.create(loc, imag, fmf); + Value scaledExp = rewriter.create(loc, half, exp, fmf); + Value reciprocalExp = rewriter.create(loc, half, exp, fmf); + Value sin = rewriter.create(loc, real, fmf); + Value cos = rewriter.create(loc, real, fmf); auto resultPair = - combine(loc, scaledExp, reciprocalExp, sin, cos, rewriter); + combine(loc, scaledExp, reciprocalExp, sin, cos, rewriter, fmf); rewriter.replaceOpWithNewOp(op, type, resultPair.first, resultPair.second); @@ -223,15 +224,17 @@ struct TrigonometricOpConversion : public OpConversionPattern { virtual std::pair combine(Location loc, Value scaledExp, Value reciprocalExp, Value sin, - Value cos, ConversionPatternRewriter &rewriter) const = 0; + Value cos, ConversionPatternRewriter &rewriter, + arith::FastMathFlagsAttr fmf) const = 0; }; struct CosOpConversion : public TrigonometricOpConversion { using TrigonometricOpConversion::TrigonometricOpConversion; - std::pair - combine(Location loc, Value scaledExp, Value reciprocalExp, Value sin, - Value cos, ConversionPatternRewriter &rewriter) const override { + std::pair combine(Location loc, Value scaledExp, + Value reciprocalExp, Value sin, Value cos, + ConversionPatternRewriter &rewriter, + arith::FastMathFlagsAttr fmf) const override { // Complex cosine is defined as; // cos(x + iy) = 0.5 * (exp(i(x + iy)) + exp(-i(x + iy))) // Plugging in: @@ -241,10 +244,12 @@ struct CosOpConversion : public TrigonometricOpConversion { // We get: // Re(cos(x + iy)) = (0.5/t + 0.5*t) * cos x // Im(cos(x + iy)) = (0.5/t - 0.5*t) * sin x - Value sum = rewriter.create(loc, reciprocalExp, scaledExp); - Value resultReal = rewriter.create(loc, sum, cos); - Value diff = rewriter.create(loc, reciprocalExp, scaledExp); - Value resultImag = rewriter.create(loc, diff, sin); + Value sum = + rewriter.create(loc, reciprocalExp, scaledExp, fmf); + Value resultReal = rewriter.create(loc, sum, cos, fmf); + Value diff = + rewriter.create(loc, reciprocalExp, scaledExp, fmf); + Value resultImag = rewriter.create(loc, diff, sin, fmf); return {resultReal, resultImag}; } }; @@ -813,9 +818,10 @@ struct NegOpConversion : public OpConversionPattern { struct SinOpConversion : public TrigonometricOpConversion { using TrigonometricOpConversion::TrigonometricOpConversion; - std::pair - combine(Location loc, Value scaledExp, Value reciprocalExp, Value sin, - Value cos, ConversionPatternRewriter &rewriter) const override { + std::pair combine(Location loc, Value scaledExp, + Value reciprocalExp, Value sin, Value cos, + ConversionPatternRewriter &rewriter, + arith::FastMathFlagsAttr fmf) const override { // Complex sine is defined as; // sin(x + iy) = -0.5i * (exp(i(x + iy)) - exp(-i(x + iy))) // Plugging in: @@ -825,10 +831,12 @@ struct SinOpConversion : public TrigonometricOpConversion { // We get: // Re(sin(x + iy)) = (0.5*t + 0.5/t) * sin x // Im(cos(x + iy)) = (0.5*t - 0.5/t) * cos x - Value sum = rewriter.create(loc, scaledExp, reciprocalExp); - Value resultReal = rewriter.create(loc, sum, sin); - Value diff = rewriter.create(loc, scaledExp, reciprocalExp); - Value resultImag = rewriter.create(loc, diff, cos); + Value sum = + rewriter.create(loc, scaledExp, reciprocalExp, fmf); + Value resultReal = rewriter.create(loc, sum, sin, fmf); + Value diff = + rewriter.create(loc, scaledExp, reciprocalExp, fmf); + Value resultImag = rewriter.create(loc, diff, cos, fmf); return {resultReal, resultImag}; } }; diff --git a/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir b/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir index 5918ff2e0f36..bac94aae6b74 100644 --- a/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir +++ b/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir @@ -1834,3 +1834,49 @@ func.func @complex_sqrt_with_fmf(%arg: complex) -> complex { // CHECK: %[[VAR40:.*]] = arith.select %[[VAR38]], %cst, %[[VAR32]] : f32 // CHECK: %[[VAR41:.*]] = complex.create %[[VAR39]], %[[VAR40]] : complex // CHECK: return %[[VAR41]] : complex + +// ----- + +// CHECK-LABEL: func @complex_cos_with_fmf +// CHECK-SAME: %[[ARG:.*]]: complex +func.func @complex_cos_with_fmf(%arg: complex) -> complex { + %cos = complex.cos %arg fastmath : complex + return %cos : complex +} +// CHECK-DAG: %[[REAL:.*]] = complex.re %[[ARG]] +// CHECK-DAG: %[[IMAG:.*]] = complex.im %[[ARG]] +// CHECK-DAG: %[[HALF:.*]] = arith.constant 5.000000e-01 : f32 +// CHECK-DAG: %[[EXP:.*]] = math.exp %[[IMAG]] fastmath : f32 +// CHECK-DAG: %[[HALF_EXP:.*]] = arith.mulf %[[HALF]], %[[EXP]] fastmath +// CHECK-DAG: %[[HALF_REXP:.*]] = arith.divf %[[HALF]], %[[EXP]] fastmath +// CHECK-DAG: %[[SIN:.*]] = math.sin %[[REAL]] fastmath : f32 +// CHECK-DAG: %[[COS:.*]] = math.cos %[[REAL]] fastmath : f32 +// CHECK-DAG: %[[EXP_SUM:.*]] = arith.addf %[[HALF_REXP]], %[[HALF_EXP]] fastmath +// CHECK-DAG: %[[RESULT_REAL:.*]] = arith.mulf %[[EXP_SUM]], %[[COS]] fastmath +// CHECK-DAG: %[[EXP_DIFF:.*]] = arith.subf %[[HALF_REXP]], %[[HALF_EXP]] fastmath +// CHECK-DAG: %[[RESULT_IMAG:.*]] = arith.mulf %[[EXP_DIFF]], %[[SIN]] fastmath +// CHECK-DAG: %[[RESULT:.*]] = complex.create %[[RESULT_REAL]], %[[RESULT_IMAG]] : complex +// CHECK: return %[[RESULT]] + +// ----- + +// CHECK-LABEL: func @complex_sin_with_fmf +// CHECK-SAME: %[[ARG:.*]]: complex +func.func @complex_sin_with_fmf(%arg: complex) -> complex { + %cos = complex.sin %arg fastmath : complex + return %cos : complex +} +// CHECK-DAG: %[[REAL:.*]] = complex.re %[[ARG]] +// CHECK-DAG: %[[IMAG:.*]] = complex.im %[[ARG]] +// CHECK-DAG: %[[HALF:.*]] = arith.constant 5.000000e-01 : f32 +// CHECK-DAG: %[[EXP:.*]] = math.exp %[[IMAG]] fastmath : f32 +// CHECK-DAG: %[[HALF_EXP:.*]] = arith.mulf %[[HALF]], %[[EXP]] fastmath +// CHECK-DAG: %[[HALF_REXP:.*]] = arith.divf %[[HALF]], %[[EXP]] fastmath +// CHECK-DAG: %[[SIN:.*]] = math.sin %[[REAL]] fastmath : f32 +// CHECK-DAG: %[[COS:.*]] = math.cos %[[REAL]] fastmath : f32 +// CHECK-DAG: %[[EXP_SUM:.*]] = arith.addf %[[HALF_EXP]], %[[HALF_REXP]] fastmath +// CHECK-DAG: %[[RESULT_REAL:.*]] = arith.mulf %[[EXP_SUM]], %[[SIN]] fastmath +// CHECK-DAG: %[[EXP_DIFF:.*]] = arith.subf %[[HALF_EXP]], %[[HALF_REXP]] fastmath +// CHECK-DAG: %[[RESULT_IMAG:.*]] = arith.mulf %[[EXP_DIFF]], %[[COS]] fastmath +// CHECK-DAG: %[[RESULT:.*]] = complex.create %[[RESULT_REAL]], %[[RESULT_IMAG]] : complex +// CHECK: return %[[RESULT]] -- GitLab From 2e4e04c59043a645572cf548de5d9c333a5d6641 Mon Sep 17 00:00:00 2001 From: Phoebe Wang Date: Mon, 25 Mar 2024 10:06:12 +0800 Subject: [PATCH 180/306] [X86][BF16] Do not lower to VCVTNEPS2BF16 without AVX512VL (#86395) Fixes: #86305 --- llvm/lib/Target/X86/X86ISelLowering.cpp | 7 ++- llvm/test/CodeGen/X86/pr86305.ll | 74 +++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 2 deletions(-) create mode 100644 llvm/test/CodeGen/X86/pr86305.ll diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index 35f756ea5e1d..9acbe17d0bca 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -21512,7 +21512,9 @@ SDValue X86TargetLowering::LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const { } if (VT.getScalarType() == MVT::bf16) { - if (SVT.getScalarType() == MVT::f32 && isTypeLegal(VT)) + if (SVT.getScalarType() == MVT::f32 && + ((Subtarget.hasBF16() && Subtarget.hasVLX()) || + Subtarget.hasAVXNECONVERT())) return Op; return SDValue(); } @@ -21619,7 +21621,8 @@ SDValue X86TargetLowering::LowerFP_TO_BF16(SDValue Op, SDLoc DL(Op); MVT SVT = Op.getOperand(0).getSimpleValueType(); - if (SVT == MVT::f32 && (Subtarget.hasBF16() || Subtarget.hasAVXNECONVERT())) { + if (SVT == MVT::f32 && ((Subtarget.hasBF16() && Subtarget.hasVLX()) || + Subtarget.hasAVXNECONVERT())) { SDValue Res; Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4f32, Op.getOperand(0)); Res = DAG.getNode(X86ISD::CVTNEPS2BF16, DL, MVT::v8bf16, Res); diff --git a/llvm/test/CodeGen/X86/pr86305.ll b/llvm/test/CodeGen/X86/pr86305.ll new file mode 100644 index 000000000000..79b42bb2532c --- /dev/null +++ b/llvm/test/CodeGen/X86/pr86305.ll @@ -0,0 +1,74 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc < %s -mtriple=x86_64-linux-gnu -mattr=avx512bf16 | FileCheck %s + +define void @add(ptr %pa, ptr %pb, ptr %pc) nounwind { +; CHECK-LABEL: add: +; CHECK: # %bb.0: +; CHECK-NEXT: pushq %rbx +; CHECK-NEXT: movq %rdx, %rbx +; CHECK-NEXT: movzwl (%rsi), %eax +; CHECK-NEXT: shll $16, %eax +; CHECK-NEXT: vmovd %eax, %xmm0 +; CHECK-NEXT: movzwl (%rdi), %eax +; CHECK-NEXT: shll $16, %eax +; CHECK-NEXT: vmovd %eax, %xmm1 +; CHECK-NEXT: vaddss %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: callq __truncsfbf2@PLT +; CHECK-NEXT: vpextrw $0, %xmm0, (%rbx) +; CHECK-NEXT: popq %rbx +; CHECK-NEXT: retq + %a = load bfloat, ptr %pa + %b = load bfloat, ptr %pb + %add = fadd bfloat %a, %b + store bfloat %add, ptr %pc + ret void +} + +define <4 x bfloat> @fptrunc_v4f32(<4 x float> %a) nounwind { +; CHECK-LABEL: fptrunc_v4f32: +; CHECK: # %bb.0: +; CHECK-NEXT: pushq %rbp +; CHECK-NEXT: pushq %r15 +; CHECK-NEXT: pushq %r14 +; CHECK-NEXT: pushq %rbx +; CHECK-NEXT: subq $72, %rsp +; CHECK-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill +; CHECK-NEXT: callq __truncsfbf2@PLT +; CHECK-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; CHECK-NEXT: vpermilpd $1, (%rsp), %xmm0 # 16-byte Folded Reload +; CHECK-NEXT: # xmm0 = mem[1,0] +; CHECK-NEXT: callq __truncsfbf2@PLT +; CHECK-NEXT: vmovapd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; CHECK-NEXT: vpshufd $255, (%rsp), %xmm0 # 16-byte Folded Reload +; CHECK-NEXT: # xmm0 = mem[3,3,3,3] +; CHECK-NEXT: callq __truncsfbf2@PLT +; CHECK-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; CHECK-NEXT: callq __truncsfbf2@PLT +; CHECK-NEXT: vpextrw $0, %xmm0, %ebx +; CHECK-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; CHECK-NEXT: vpextrw $0, %xmm0, %ebp +; CHECK-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; CHECK-NEXT: vpextrw $0, %xmm0, %r14d +; CHECK-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; CHECK-NEXT: vpextrw $0, %xmm0, %r15d +; CHECK-NEXT: vmovshdup (%rsp), %xmm0 # 16-byte Folded Reload +; CHECK-NEXT: # xmm0 = mem[1,1,3,3] +; CHECK-NEXT: callq __truncsfbf2@PLT +; CHECK-NEXT: vpextrw $0, %xmm0, %eax +; CHECK-NEXT: vmovd %r15d, %xmm0 +; CHECK-NEXT: vpinsrw $1, %eax, %xmm0, %xmm0 +; CHECK-NEXT: vpinsrw $2, %r14d, %xmm0, %xmm0 +; CHECK-NEXT: vpinsrw $3, %ebp, %xmm0, %xmm0 +; CHECK-NEXT: vpinsrw $4, %ebx, %xmm0, %xmm0 +; CHECK-NEXT: vpinsrw $5, %ebx, %xmm0, %xmm0 +; CHECK-NEXT: vpinsrw $6, %ebx, %xmm0, %xmm0 +; CHECK-NEXT: vpinsrw $7, %ebx, %xmm0, %xmm0 +; CHECK-NEXT: addq $72, %rsp +; CHECK-NEXT: popq %rbx +; CHECK-NEXT: popq %r14 +; CHECK-NEXT: popq %r15 +; CHECK-NEXT: popq %rbp +; CHECK-NEXT: retq + %b = fptrunc <4 x float> %a to <4 x bfloat> + ret <4 x bfloat> %b +} -- GitLab From 5e5b6561029665e69e033cff4216fecb78302259 Mon Sep 17 00:00:00 2001 From: Sergei Barannikov Date: Mon, 25 Mar 2024 05:13:48 +0300 Subject: [PATCH 181/306] [MC] Make `MCParsedAsmOperand::getReg()` return `MCRegister` (#86444) --- .../llvm/MC/MCParser/MCParsedAsmOperand.h | 3 ++- .../llvm/MC/MCParser/MCTargetAsmParser.h | 4 +-- llvm/lib/MC/MCParser/MCTargetAsmParser.cpp | 6 +++++ .../AArch64/AsmParser/AArch64AsmParser.cpp | 2 +- .../AMDGPU/AsmParser/AMDGPUAsmParser.cpp | 2 +- .../lib/Target/ARM/AsmParser/ARMAsmParser.cpp | 2 +- .../lib/Target/AVR/AsmParser/AVRAsmParser.cpp | 2 +- .../lib/Target/BPF/AsmParser/BPFAsmParser.cpp | 2 +- .../Target/CSKY/AsmParser/CSKYAsmParser.cpp | 2 +- .../Hexagon/AsmParser/HexagonAsmParser.cpp | 2 +- .../Target/Lanai/AsmParser/LanaiAsmParser.cpp | 2 +- .../AsmParser/LoongArchAsmParser.cpp | 4 +-- .../Target/M68k/AsmParser/M68kAsmParser.cpp | 4 +-- .../MSP430/AsmParser/MSP430AsmParser.cpp | 2 +- .../Target/Mips/AsmParser/MipsAsmParser.cpp | 2 +- .../Target/PowerPC/AsmParser/PPCAsmParser.cpp | 26 ++++++++++--------- .../Target/RISCV/AsmParser/RISCVAsmParser.cpp | 4 +-- .../Target/Sparc/AsmParser/SparcAsmParser.cpp | 2 +- .../SystemZ/AsmParser/SystemZAsmParser.cpp | 2 +- llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp | 2 +- .../AsmParser/WebAssemblyAsmParser.cpp | 2 +- llvm/lib/Target/X86/AsmParser/X86Operand.h | 2 +- .../Xtensa/AsmParser/XtensaAsmParser.cpp | 2 +- llvm/utils/TableGen/AsmMatcherEmitter.cpp | 2 +- 24 files changed, 46 insertions(+), 39 deletions(-) diff --git a/llvm/include/llvm/MC/MCParser/MCParsedAsmOperand.h b/llvm/include/llvm/MC/MCParser/MCParsedAsmOperand.h index 0c9668904e82..27ecb7b85d22 100644 --- a/llvm/include/llvm/MC/MCParser/MCParsedAsmOperand.h +++ b/llvm/include/llvm/MC/MCParser/MCParsedAsmOperand.h @@ -15,6 +15,7 @@ namespace llvm { +class MCRegister; class raw_ostream; /// MCParsedAsmOperand - This abstract class represents a source-level assembly @@ -57,7 +58,7 @@ public: virtual bool isImm() const = 0; /// isReg - Is this a register operand? virtual bool isReg() const = 0; - virtual unsigned getReg() const = 0; + virtual MCRegister getReg() const = 0; /// isMem - Is this a memory operand? virtual bool isMem() const = 0; diff --git a/llvm/include/llvm/MC/MCParser/MCTargetAsmParser.h b/llvm/include/llvm/MC/MCParser/MCTargetAsmParser.h index 7edd3f8ce490..49ce417e6fbb 100644 --- a/llvm/include/llvm/MC/MCParser/MCTargetAsmParser.h +++ b/llvm/include/llvm/MC/MCParser/MCTargetAsmParser.h @@ -514,9 +514,7 @@ public: /// by the tied-operands checks in the AsmMatcher. This method can be /// overridden to allow e.g. a sub- or super-register as the tied operand. virtual bool areEqualRegs(const MCParsedAsmOperand &Op1, - const MCParsedAsmOperand &Op2) const { - return Op1.isReg() && Op2.isReg() && Op1.getReg() == Op2.getReg(); - } + const MCParsedAsmOperand &Op2) const; // Return whether this parser uses assignment statements with equals tokens virtual bool equalIsAsmAssignment() { return true; }; diff --git a/llvm/lib/MC/MCParser/MCTargetAsmParser.cpp b/llvm/lib/MC/MCParser/MCTargetAsmParser.cpp index 0db5fb36f795..665d92eb9a21 100644 --- a/llvm/lib/MC/MCParser/MCTargetAsmParser.cpp +++ b/llvm/lib/MC/MCParser/MCTargetAsmParser.cpp @@ -8,6 +8,7 @@ #include "llvm/MC/MCParser/MCTargetAsmParser.h" #include "llvm/MC/MCContext.h" +#include "llvm/MC/MCRegister.h" using namespace llvm; @@ -48,3 +49,8 @@ ParseStatus MCTargetAsmParser::parseDirective(AsmToken DirectiveID) { return ParseStatus::Failure; return ParseStatus::NoMatch; } + +bool MCTargetAsmParser::areEqualRegs(const MCParsedAsmOperand &Op1, + const MCParsedAsmOperand &Op2) const { + return Op1.isReg() && Op2.isReg() && Op1.getReg() == Op2.getReg(); +} diff --git a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp index b807aaf76fdb..21643ebb4138 100644 --- a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp +++ b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp @@ -654,7 +654,7 @@ public: return Barrier.HasnXSModifier; } - unsigned getReg() const override { + MCRegister getReg() const override { assert(Kind == k_Register && "Invalid access!"); return Reg.RegNum; } diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp index 529705479646..4648df199c74 100644 --- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp +++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp @@ -981,7 +981,7 @@ public: return Imm.Type; } - unsigned getReg() const override { + MCRegister getReg() const override { assert(isRegKind()); return Reg.RegNo; } diff --git a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp index 9cfdb15a0f43..2ad576ab1a9c 100644 --- a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp +++ b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp @@ -1002,7 +1002,7 @@ public: return StringRef(Tok.Data, Tok.Length); } - unsigned getReg() const override { + MCRegister getReg() const override { assert((Kind == k_Register || Kind == k_CCOut) && "Invalid access!"); return Reg.RegNum; } diff --git a/llvm/lib/Target/AVR/AsmParser/AVRAsmParser.cpp b/llvm/lib/Target/AVR/AsmParser/AVRAsmParser.cpp index db4aa03437c6..383dfcc31117 100644 --- a/llvm/lib/Target/AVR/AsmParser/AVRAsmParser.cpp +++ b/llvm/lib/Target/AVR/AsmParser/AVRAsmParser.cpp @@ -195,7 +195,7 @@ public: return Tok; } - unsigned getReg() const override { + MCRegister getReg() const override { assert((Kind == k_Register || Kind == k_Memri) && "Invalid access!"); return RegImm.Reg; diff --git a/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp b/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp index 1688355f427c..9672ed009e9b 100644 --- a/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp +++ b/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp @@ -148,7 +148,7 @@ public: /// getEndLoc - Gets location of the last token of this operand SMLoc getEndLoc() const override { return EndLoc; } - unsigned getReg() const override { + MCRegister getReg() const override { assert(Kind == Register && "Invalid type access!"); return Reg.RegNum; } diff --git a/llvm/lib/Target/CSKY/AsmParser/CSKYAsmParser.cpp b/llvm/lib/Target/CSKY/AsmParser/CSKYAsmParser.cpp index 4711e58bbed6..30bd3dcefa60 100644 --- a/llvm/lib/Target/CSKY/AsmParser/CSKYAsmParser.cpp +++ b/llvm/lib/Target/CSKY/AsmParser/CSKYAsmParser.cpp @@ -400,7 +400,7 @@ public: /// Gets location of the last token of this operand. SMLoc getEndLoc() const override { return EndLoc; } - unsigned getReg() const override { + MCRegister getReg() const override { assert(Kind == Register && "Invalid type access!"); return Reg.RegNum; } diff --git a/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp b/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp index 864591d4eb95..092cccbcca9c 100644 --- a/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp +++ b/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp @@ -245,7 +245,7 @@ public: /// getEndLoc - Get the location of the last token of this operand. SMLoc getEndLoc() const override { return EndLoc; } - unsigned getReg() const override { + MCRegister getReg() const override { assert(Kind == Register && "Invalid access!"); return Reg.RegNum; } diff --git a/llvm/lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp b/llvm/lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp index ff3649b77e35..6ab1375b974e 100644 --- a/llvm/lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp +++ b/llvm/lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp @@ -151,7 +151,7 @@ public: // getEndLoc - Gets location of the last token of this operand SMLoc getEndLoc() const override { return EndLoc; } - unsigned getReg() const override { + MCRegister getReg() const override { assert(isReg() && "Invalid type access!"); return Reg.RegNum; } diff --git a/llvm/lib/Target/LoongArch/AsmParser/LoongArchAsmParser.cpp b/llvm/lib/Target/LoongArch/AsmParser/LoongArchAsmParser.cpp index cf163e4e1200..20284b18428b 100644 --- a/llvm/lib/Target/LoongArch/AsmParser/LoongArchAsmParser.cpp +++ b/llvm/lib/Target/LoongArch/AsmParser/LoongArchAsmParser.cpp @@ -467,9 +467,9 @@ public: /// Gets location of the last token of this operand. SMLoc getEndLoc() const override { return EndLoc; } - unsigned getReg() const override { + MCRegister getReg() const override { assert(Kind == KindTy::Register && "Invalid type access!"); - return Reg.RegNum.id(); + return Reg.RegNum; } const MCExpr *getImm() const { diff --git a/llvm/lib/Target/M68k/AsmParser/M68kAsmParser.cpp b/llvm/lib/Target/M68k/AsmParser/M68kAsmParser.cpp index b2c0fda1ccc2..126176133dc0 100644 --- a/llvm/lib/Target/M68k/AsmParser/M68kAsmParser.cpp +++ b/llvm/lib/Target/M68k/AsmParser/M68kAsmParser.cpp @@ -157,7 +157,7 @@ public: bool isDReg() const; bool isFPDReg() const; bool isFPCReg() const; - unsigned getReg() const override; + MCRegister getReg() const override; void addRegOperands(MCInst &Inst, unsigned N) const; static std::unique_ptr createMemOp(M68kMemOp MemOp, SMLoc Start, @@ -312,7 +312,7 @@ bool M68kOperand::isReg() const { return Kind == KindTy::MemOp && MemOp.Op == M68kMemOp::Kind::Reg; } -unsigned M68kOperand::getReg() const { +MCRegister M68kOperand::getReg() const { assert(isReg()); return MemOp.OuterReg; } diff --git a/llvm/lib/Target/MSP430/AsmParser/MSP430AsmParser.cpp b/llvm/lib/Target/MSP430/AsmParser/MSP430AsmParser.cpp index 818a468612a5..2bc1a89ef59c 100644 --- a/llvm/lib/Target/MSP430/AsmParser/MSP430AsmParser.cpp +++ b/llvm/lib/Target/MSP430/AsmParser/MSP430AsmParser.cpp @@ -183,7 +183,7 @@ public: return Tok; } - unsigned getReg() const override { + MCRegister getReg() const override { assert(Kind == k_Reg && "Invalid access!"); return Reg; } diff --git a/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp b/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp index 9d6e8dc573a8..076e0a20cb97 100644 --- a/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp +++ b/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp @@ -1458,7 +1458,7 @@ public: return StringRef(Tok.Data, Tok.Length); } - unsigned getReg() const override { + MCRegister getReg() const override { // As a special case until we sort out the definition of div/divu, accept // $0/$zero here so that MCK_ZERO works correctly. if (Kind == k_RegisterIndex && RegIdx.Index == 0 && diff --git a/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp b/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp index 8108cfa521c8..55978af38000 100644 --- a/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp +++ b/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp @@ -276,9 +276,11 @@ public: return TLSReg.Sym; } - unsigned getReg() const override { + MCRegister getReg() const override { llvm_unreachable("Not implemented"); } + + unsigned getRegNum() const { assert(isRegNumber() && "Invalid access!"); - return (unsigned) Imm.Val; + return (unsigned)Imm.Val; } unsigned getFpReg() const { @@ -459,22 +461,22 @@ public: void addRegGPRCOperands(MCInst &Inst, unsigned N) const { assert(N == 1 && "Invalid number of operands!"); - Inst.addOperand(MCOperand::createReg(RRegs[getReg()])); + Inst.addOperand(MCOperand::createReg(RRegs[getRegNum()])); } void addRegGPRCNoR0Operands(MCInst &Inst, unsigned N) const { assert(N == 1 && "Invalid number of operands!"); - Inst.addOperand(MCOperand::createReg(RRegsNoR0[getReg()])); + Inst.addOperand(MCOperand::createReg(RRegsNoR0[getRegNum()])); } void addRegG8RCOperands(MCInst &Inst, unsigned N) const { assert(N == 1 && "Invalid number of operands!"); - Inst.addOperand(MCOperand::createReg(XRegs[getReg()])); + Inst.addOperand(MCOperand::createReg(XRegs[getRegNum()])); } void addRegG8RCNoX0Operands(MCInst &Inst, unsigned N) const { assert(N == 1 && "Invalid number of operands!"); - Inst.addOperand(MCOperand::createReg(XRegsNoX0[getReg()])); + Inst.addOperand(MCOperand::createReg(XRegsNoX0[getRegNum()])); } void addRegG8pRCOperands(MCInst &Inst, unsigned N) const { @@ -498,12 +500,12 @@ public: void addRegF4RCOperands(MCInst &Inst, unsigned N) const { assert(N == 1 && "Invalid number of operands!"); - Inst.addOperand(MCOperand::createReg(FRegs[getReg()])); + Inst.addOperand(MCOperand::createReg(FRegs[getRegNum()])); } void addRegF8RCOperands(MCInst &Inst, unsigned N) const { assert(N == 1 && "Invalid number of operands!"); - Inst.addOperand(MCOperand::createReg(FRegs[getReg()])); + Inst.addOperand(MCOperand::createReg(FRegs[getRegNum()])); } void addRegFpRCOperands(MCInst &Inst, unsigned N) const { @@ -513,12 +515,12 @@ public: void addRegVFRCOperands(MCInst &Inst, unsigned N) const { assert(N == 1 && "Invalid number of operands!"); - Inst.addOperand(MCOperand::createReg(VFRegs[getReg()])); + Inst.addOperand(MCOperand::createReg(VFRegs[getRegNum()])); } void addRegVRRCOperands(MCInst &Inst, unsigned N) const { assert(N == 1 && "Invalid number of operands!"); - Inst.addOperand(MCOperand::createReg(VRegs[getReg()])); + Inst.addOperand(MCOperand::createReg(VRegs[getRegNum()])); } void addRegVSRCOperands(MCInst &Inst, unsigned N) const { @@ -538,12 +540,12 @@ public: void addRegSPE4RCOperands(MCInst &Inst, unsigned N) const { assert(N == 1 && "Invalid number of operands!"); - Inst.addOperand(MCOperand::createReg(RRegs[getReg()])); + Inst.addOperand(MCOperand::createReg(RRegs[getRegNum()])); } void addRegSPERCOperands(MCInst &Inst, unsigned N) const { assert(N == 1 && "Invalid number of operands!"); - Inst.addOperand(MCOperand::createReg(SPERegs[getReg()])); + Inst.addOperand(MCOperand::createReg(SPERegs[getRegNum()])); } void addRegACCRCOperands(MCInst &Inst, unsigned N) const { diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp index cb2ba52390e2..5e594d6cad6b 100644 --- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp +++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp @@ -977,9 +977,9 @@ public: return Imm.IsRV64; } - unsigned getReg() const override { + MCRegister getReg() const override { assert(Kind == KindTy::Register && "Invalid type access!"); - return Reg.RegNum.id(); + return Reg.RegNum; } StringRef getSysReg() const { diff --git a/llvm/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp b/llvm/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp index be4ec1e9dce2..67e2b9d7c997 100644 --- a/llvm/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp +++ b/llvm/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp @@ -307,7 +307,7 @@ public: return StringRef(Tok.Data, Tok.Length); } - unsigned getReg() const override { + MCRegister getReg() const override { assert((Kind == k_Register) && "Invalid access!"); return Reg.RegNum; } diff --git a/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp b/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp index a58e8e0dfedf..f2c04215d12d 100644 --- a/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp +++ b/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp @@ -227,7 +227,7 @@ public: bool isReg(RegisterKind RegKind) const { return Kind == KindReg && Reg.Kind == RegKind; } - unsigned getReg() const override { + MCRegister getReg() const override { assert(Kind == KindReg && "Not a register"); return Reg.Num; } diff --git a/llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp b/llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp index f9e30a3a9378..691fe8fe3aa4 100644 --- a/llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp +++ b/llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp @@ -344,7 +344,7 @@ public: return StringRef(Tok.Data, Tok.Length); } - unsigned getReg() const override { + MCRegister getReg() const override { assert((Kind == k_Register) && "Invalid access!"); return Reg.RegNum; } diff --git a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp index 3cc4d50271eb..020c0d6229d2 100644 --- a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp +++ b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp @@ -100,7 +100,7 @@ struct WebAssemblyOperand : public MCParsedAsmOperand { bool isReg() const override { return false; } bool isBrList() const { return Kind == BrList; } - unsigned getReg() const override { + MCRegister getReg() const override { llvm_unreachable("Assembly inspects a register operand"); return 0; } diff --git a/llvm/lib/Target/X86/AsmParser/X86Operand.h b/llvm/lib/Target/X86/AsmParser/X86Operand.h index 641158cb351f..78669784dd03 100644 --- a/llvm/lib/Target/X86/AsmParser/X86Operand.h +++ b/llvm/lib/Target/X86/AsmParser/X86Operand.h @@ -167,7 +167,7 @@ struct X86Operand final : public MCParsedAsmOperand { Tok.Length = Value.size(); } - unsigned getReg() const override { + MCRegister getReg() const override { assert(Kind == Register && "Invalid access!"); return Reg.RegNo; } diff --git a/llvm/lib/Target/Xtensa/AsmParser/XtensaAsmParser.cpp b/llvm/lib/Target/Xtensa/AsmParser/XtensaAsmParser.cpp index 3f808298527f..1fa00af2111e 100644 --- a/llvm/lib/Target/Xtensa/AsmParser/XtensaAsmParser.cpp +++ b/llvm/lib/Target/Xtensa/AsmParser/XtensaAsmParser.cpp @@ -244,7 +244,7 @@ public: /// getEndLoc - Gets location of the last token of this operand SMLoc getEndLoc() const override { return EndLoc; } - unsigned getReg() const override { + MCRegister getReg() const override { assert(Kind == Register && "Invalid type access!"); return Reg.RegNum; } diff --git a/llvm/utils/TableGen/AsmMatcherEmitter.cpp b/llvm/utils/TableGen/AsmMatcherEmitter.cpp index 5df7990d8fc2..0a2b1cf0f9a6 100644 --- a/llvm/utils/TableGen/AsmMatcherEmitter.cpp +++ b/llvm/utils/TableGen/AsmMatcherEmitter.cpp @@ -2519,7 +2519,7 @@ static void emitValidateOperandClass(AsmMatcherInfo &Info, raw_ostream &OS) { // Check for register operands, including sub-classes. OS << " if (Operand.isReg()) {\n"; OS << " MatchClassKind OpKind;\n"; - OS << " switch (Operand.getReg()) {\n"; + OS << " switch (Operand.getReg().id()) {\n"; OS << " default: OpKind = InvalidMatchClass; break;\n"; for (const auto &RC : Info.RegisterClasses) OS << " case " << RC.first->getValueAsString("Namespace") -- GitLab From 6af6416e89de1f4656a145c4843226b468718434 Mon Sep 17 00:00:00 2001 From: Wang Pengcheng Date: Mon, 25 Mar 2024 11:44:16 +0800 Subject: [PATCH 182/306] [RISCV] Add a tune feature to disable stripping W suffix (#86255) We have a hidden option to disable it, but I'd like to make it a tune feature. For some implementations, instructions with W suffix would be less costly as they only perform on 32 bits data. Though we may lose some chances to compress. --- llvm/lib/Target/RISCV/RISCVFeatures.td | 4 ++ llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp | 2 +- llvm/test/CodeGen/RISCV/strip-w-suffix.ll | 74 +++++++++++++++++++++++ 3 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 llvm/test/CodeGen/RISCV/strip-w-suffix.ll diff --git a/llvm/lib/Target/RISCV/RISCVFeatures.td b/llvm/lib/Target/RISCV/RISCVFeatures.td index f3e641e25018..6ef2289bb4be 100644 --- a/llvm/lib/Target/RISCV/RISCVFeatures.td +++ b/llvm/lib/Target/RISCV/RISCVFeatures.td @@ -1226,6 +1226,10 @@ def TuneNoSinkSplatOperands "false", "Disable sink splat operands to enable .vx, .vf," ".wx, and .wf instructions">; +def TuneNoStripWSuffix + : SubtargetFeature<"no-strip-w-suffix", "EnableStripWSuffix", "false", + "Disable strip W suffix">; + def TuneConditionalCompressedMoveFusion : SubtargetFeature<"conditional-cmv-fusion", "HasConditionalCompressedMoveFusion", "true", "Enable branch+c.mv fusion">; diff --git a/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp b/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp index dcf70e8cad64..39d420c2fbf0 100644 --- a/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp +++ b/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp @@ -672,7 +672,7 @@ bool RISCVOptWInstrs::stripWSuffixes(MachineFunction &MF, const RISCVInstrInfo &TII, const RISCVSubtarget &ST, MachineRegisterInfo &MRI) { - if (DisableStripWSuffix) + if (DisableStripWSuffix || !ST.enableStripWSuffix()) return false; bool MadeChange = false; diff --git a/llvm/test/CodeGen/RISCV/strip-w-suffix.ll b/llvm/test/CodeGen/RISCV/strip-w-suffix.ll new file mode 100644 index 000000000000..4124b3d0d360 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/strip-w-suffix.ll @@ -0,0 +1,74 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -mtriple=riscv64 -mattr=+m -verify-machineinstrs < %s \ +; RUN: | FileCheck -check-prefixes=STRIP %s +; RUN: llc -mtriple=riscv64 -mattr=+m,+no-strip-w-suffix -verify-machineinstrs < %s \ +; RUN: | FileCheck -check-prefixes=NO-STRIP %s + +define i32 @addiw(i32 %a) { +; STRIP-LABEL: addiw: +; STRIP: # %bb.0: +; STRIP-NEXT: lui a1, 1 +; STRIP-NEXT: addi a1, a1, -1 +; STRIP-NEXT: addw a0, a0, a1 +; STRIP-NEXT: ret +; +; NO-STRIP-LABEL: addiw: +; NO-STRIP: # %bb.0: +; NO-STRIP-NEXT: lui a1, 1 +; NO-STRIP-NEXT: addiw a1, a1, -1 +; NO-STRIP-NEXT: addw a0, a0, a1 +; NO-STRIP-NEXT: ret + %ret = add i32 %a, 4095 + ret i32 %ret +} + +define i32 @addw(i32 %a, i32 %b) { +; STRIP-LABEL: addw: +; STRIP: # %bb.0: +; STRIP-NEXT: add a0, a0, a1 +; STRIP-NEXT: addiw a0, a0, 1024 +; STRIP-NEXT: ret +; +; NO-STRIP-LABEL: addw: +; NO-STRIP: # %bb.0: +; NO-STRIP-NEXT: addw a0, a0, a1 +; NO-STRIP-NEXT: addiw a0, a0, 1024 +; NO-STRIP-NEXT: ret + %add = add i32 %a, %b + %ret = add i32 %add, 1024 + ret i32 %ret +} + +define i32 @mulw(i32 %a, i32 %b) { +; STRIP-LABEL: mulw: +; STRIP: # %bb.0: +; STRIP-NEXT: mul a0, a0, a1 +; STRIP-NEXT: addiw a0, a0, 1024 +; STRIP-NEXT: ret +; +; NO-STRIP-LABEL: mulw: +; NO-STRIP: # %bb.0: +; NO-STRIP-NEXT: mulw a0, a0, a1 +; NO-STRIP-NEXT: addiw a0, a0, 1024 +; NO-STRIP-NEXT: ret + %mul = mul i32 %a, %b + %ret = add i32 %mul, 1024 + ret i32 %ret +} + +define i32 @slliw(i32 %a) { +; STRIP-LABEL: slliw: +; STRIP: # %bb.0: +; STRIP-NEXT: slli a0, a0, 1 +; STRIP-NEXT: addiw a0, a0, 1024 +; STRIP-NEXT: ret +; +; NO-STRIP-LABEL: slliw: +; NO-STRIP: # %bb.0: +; NO-STRIP-NEXT: slliw a0, a0, 1 +; NO-STRIP-NEXT: addiw a0, a0, 1024 +; NO-STRIP-NEXT: ret + %shl = shl i32 %a, 1 + %ret = add i32 %shl, 1024 + ret i32 %ret +} -- GitLab From d9746a6a5d523e21eee2c1b50c2f08aa19396965 Mon Sep 17 00:00:00 2001 From: Wang Pengcheng Date: Mon, 25 Mar 2024 12:42:59 +0800 Subject: [PATCH 183/306] [RISCV][NFC] Pass LMUL to copyPhysRegVector The opcode will be determined by LMUL. Reviewers: preames, lukel97, topperc Reviewed By: lukel97, topperc Pull Request: https://github.com/llvm/llvm-project/pull/84448 --- llvm/lib/Target/RISCV/RISCVInstrInfo.cpp | 52 ++++++++++++------------ llvm/lib/Target/RISCV/RISCVInstrInfo.h | 2 +- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp index 4ca300f9151e..14c2d41e80f1 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp @@ -299,36 +299,36 @@ void RISCVInstrInfo::copyPhysRegVector(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, MCRegister DstReg, MCRegister SrcReg, bool KillSrc, - unsigned Opc, unsigned NF) const { + RISCVII::VLMUL LMul, unsigned NF) const { const TargetRegisterInfo *TRI = STI.getRegisterInfo(); - RISCVII::VLMUL LMul; + unsigned Opc; unsigned SubRegIdx; unsigned VVOpc, VIOpc; - switch (Opc) { + switch (LMul) { default: llvm_unreachable("Impossible LMUL for vector register copy."); - case RISCV::VMV1R_V: - LMul = RISCVII::LMUL_1; + case RISCVII::LMUL_1: + Opc = RISCV::VMV1R_V; SubRegIdx = RISCV::sub_vrm1_0; VVOpc = RISCV::PseudoVMV_V_V_M1; VIOpc = RISCV::PseudoVMV_V_I_M1; break; - case RISCV::VMV2R_V: - LMul = RISCVII::LMUL_2; + case RISCVII::LMUL_2: + Opc = RISCV::VMV2R_V; SubRegIdx = RISCV::sub_vrm2_0; VVOpc = RISCV::PseudoVMV_V_V_M2; VIOpc = RISCV::PseudoVMV_V_I_M2; break; - case RISCV::VMV4R_V: - LMul = RISCVII::LMUL_4; + case RISCVII::LMUL_4: + Opc = RISCV::VMV4R_V; SubRegIdx = RISCV::sub_vrm4_0; VVOpc = RISCV::PseudoVMV_V_V_M4; VIOpc = RISCV::PseudoVMV_V_I_M4; break; - case RISCV::VMV8R_V: + case RISCVII::LMUL_8: assert(NF == 1); - LMul = RISCVII::LMUL_8; + Opc = RISCV::VMV8R_V; SubRegIdx = RISCV::sub_vrm1_0; // There is no sub_vrm8_0. VVOpc = RISCV::PseudoVMV_V_V_M8; VIOpc = RISCV::PseudoVMV_V_I_M8; @@ -505,87 +505,87 @@ void RISCVInstrInfo::copyPhysReg(MachineBasicBlock &MBB, // VR->VR copies. if (RISCV::VRRegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV1R_V); + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_1); return; } if (RISCV::VRM2RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV2R_V); + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_2); return; } if (RISCV::VRM4RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV4R_V); + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_4); return; } if (RISCV::VRM8RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV8R_V); + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_8); return; } if (RISCV::VRN2M1RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV1R_V, + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_1, /*NF=*/2); return; } if (RISCV::VRN2M2RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV2R_V, + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_2, /*NF=*/2); return; } if (RISCV::VRN2M4RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV4R_V, + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_4, /*NF=*/2); return; } if (RISCV::VRN3M1RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV1R_V, + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_1, /*NF=*/3); return; } if (RISCV::VRN3M2RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV2R_V, + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_2, /*NF=*/3); return; } if (RISCV::VRN4M1RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV1R_V, + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_1, /*NF=*/4); return; } if (RISCV::VRN4M2RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV2R_V, + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_2, /*NF=*/4); return; } if (RISCV::VRN5M1RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV1R_V, + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_1, /*NF=*/5); return; } if (RISCV::VRN6M1RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV1R_V, + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_1, /*NF=*/6); return; } if (RISCV::VRN7M1RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV1R_V, + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_1, /*NF=*/7); return; } if (RISCV::VRN8M1RegClass.contains(DstReg, SrcReg)) { - copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV1R_V, + copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_1, /*NF=*/8); return; } diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.h b/llvm/lib/Target/RISCV/RISCVInstrInfo.h index 8a312ee5e779..dd049fca0597 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfo.h +++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.h @@ -69,7 +69,7 @@ public: void copyPhysRegVector(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, MCRegister DstReg, MCRegister SrcReg, bool KillSrc, - unsigned Opc, unsigned NF = 1) const; + RISCVII::VLMUL LMul, unsigned NF = 1) const; void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, MCRegister DstReg, MCRegister SrcReg, bool KillSrc) const override; -- GitLab From 373e77b4c0ad9b0bf370f0e5a32a4100a5459d82 Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Mon, 25 Mar 2024 13:08:56 +0800 Subject: [PATCH 184/306] [RISCV] Generalize (sub zext, zext) -> (sext (sub zext, zext)) to add (#86248) This generalizes the combine added in #82455 to other binary ops, beginning with adds in this patch. Because the two zext operands are always +ve when treated as signed, and we don't get any overflow since the add is carried out in at least N * 2 bits of the narrow type, the result of the add will always be +ve. So we can use a zext for the outer extend, unlike sub which may produce a -ve result from two +ve operands. Although we could still use sext for add, I plan to add support for other binary ops like mul in a later patch, but mul requires zext to be correct (because the maximum value will take up the full N * 2 bits). So I've opted to use zext here too for consistency. Alive2 proof: https://alive2.llvm.org/ce/z/PRNsUM --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 75 +++++++--- .../CodeGen/RISCV/rvv/fixed-vectors-vwaddu.ll | 32 ++--- llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll | 128 ++++++++---------- 3 files changed, 125 insertions(+), 110 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 5214595485ca..4ace50aa477a 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -12913,6 +12913,55 @@ static SDValue transformAddImmMulImm(SDNode *N, SelectionDAG &DAG, return DAG.getNode(ISD::ADD, DL, VT, New1, DAG.getConstant(CB, DL, VT)); } +// add (zext, zext) -> zext (add (zext, zext)) +// sub (zext, zext) -> sext (sub (zext, zext)) +// +// where the sum of the extend widths match, and the the range of the bin op +// fits inside the width of the narrower bin op. (For profitability on rvv, we +// use a power of two for both inner and outer extend.) +// +// TODO: Extend this to other binary ops +static SDValue combineBinOpOfZExt(SDNode *N, SelectionDAG &DAG) { + + EVT VT = N->getValueType(0); + if (!VT.isVector() || !DAG.getTargetLoweringInfo().isTypeLegal(VT)) + return SDValue(); + + SDValue N0 = N->getOperand(0); + SDValue N1 = N->getOperand(1); + if (N0.getOpcode() != ISD::ZERO_EXTEND || N1.getOpcode() != ISD::ZERO_EXTEND) + return SDValue(); + if (!N0.hasOneUse() || !N1.hasOneUse()) + return SDValue(); + + SDValue Src0 = N0.getOperand(0); + SDValue Src1 = N1.getOperand(0); + EVT SrcVT = Src0.getValueType(); + if (!DAG.getTargetLoweringInfo().isTypeLegal(SrcVT) || + SrcVT != Src1.getValueType() || SrcVT.getScalarSizeInBits() < 8 || + SrcVT.getScalarSizeInBits() >= VT.getScalarSizeInBits() / 2) + return SDValue(); + + LLVMContext &C = *DAG.getContext(); + EVT ElemVT = VT.getVectorElementType().getHalfSizedIntegerVT(C); + EVT NarrowVT = EVT::getVectorVT(C, ElemVT, VT.getVectorElementCount()); + + Src0 = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(Src0), NarrowVT, Src0); + Src1 = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(Src1), NarrowVT, Src1); + + // Src0 and Src1 are zero extended, so they're always positive if signed. + // + // sub can produce a negative from two positive operands, so it needs sign + // extended. Other nodes produce a positive from two positive operands, so + // zero extend instead. + unsigned OuterExtend = + N->getOpcode() == ISD::SUB ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; + + return DAG.getNode( + OuterExtend, SDLoc(N), VT, + DAG.getNode(N->getOpcode(), SDLoc(N), NarrowVT, Src0, Src1)); +} + // Try to turn (add (xor bool, 1) -1) into (neg bool). static SDValue combineAddOfBooleanXor(SDNode *N, SelectionDAG &DAG) { SDValue N0 = N->getOperand(0); @@ -12950,6 +12999,8 @@ static SDValue performADDCombine(SDNode *N, SelectionDAG &DAG, return V; if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget)) return V; + if (SDValue V = combineBinOpOfZExt(N, DAG)) + return V; // fold (add (select lhs, rhs, cc, 0, y), x) -> // (select lhs, rhs, cc, x, (add x, y)) @@ -13017,28 +13068,8 @@ static SDValue performSUBCombine(SDNode *N, SelectionDAG &DAG, } } - // sub (zext, zext) -> sext (sub (zext, zext)) - // where the sum of the extend widths match, and the inner zexts - // add at least one bit. (For profitability on rvv, we use a - // power of two for both inner and outer extend.) - if (VT.isVector() && Subtarget.getTargetLowering()->isTypeLegal(VT) && - N0.getOpcode() == N1.getOpcode() && N0.getOpcode() == ISD::ZERO_EXTEND && - N0.hasOneUse() && N1.hasOneUse()) { - SDValue Src0 = N0.getOperand(0); - SDValue Src1 = N1.getOperand(0); - EVT SrcVT = Src0.getValueType(); - if (Subtarget.getTargetLowering()->isTypeLegal(SrcVT) && - SrcVT == Src1.getValueType() && SrcVT.getScalarSizeInBits() >= 8 && - SrcVT.getScalarSizeInBits() < VT.getScalarSizeInBits() / 2) { - LLVMContext &C = *DAG.getContext(); - EVT ElemVT = VT.getVectorElementType().getHalfSizedIntegerVT(C); - EVT NarrowVT = EVT::getVectorVT(C, ElemVT, VT.getVectorElementCount()); - Src0 = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(Src0), NarrowVT, Src0); - Src1 = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(Src1), NarrowVT, Src1); - return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, - DAG.getNode(ISD::SUB, SDLoc(N), NarrowVT, Src0, Src1)); - } - } + if (SDValue V = combineBinOpOfZExt(N, DAG)) + return V; // fold (sub x, (select lhs, rhs, cc, 0, y)) -> // (select lhs, rhs, cc, x, (sub x, y)) diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwaddu.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwaddu.ll index 57a72c639b33..bc0bf5dd76ad 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwaddu.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwaddu.ll @@ -385,12 +385,12 @@ define <32 x i64> @vwaddu_v32i64(ptr %x, ptr %y) nounwind { define <2 x i32> @vwaddu_v2i32_v2i8(ptr %x, ptr %y) { ; CHECK-LABEL: vwaddu_v2i32_v2i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetivli zero, 2, e16, mf4, ta, ma +; CHECK-NEXT: vsetivli zero, 2, e8, mf8, ta, ma ; CHECK-NEXT: vle8.v v8, (a0) ; CHECK-NEXT: vle8.v v9, (a1) -; CHECK-NEXT: vzext.vf2 v10, v8 -; CHECK-NEXT: vzext.vf2 v11, v9 -; CHECK-NEXT: vwaddu.vv v8, v10, v11 +; CHECK-NEXT: vwaddu.vv v10, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf2 v8, v10 ; CHECK-NEXT: ret %a = load <2 x i8>, ptr %x %b = load <2 x i8>, ptr %y @@ -912,12 +912,12 @@ define <4 x i64> @crash(<4 x i16> %x, <4 x i16> %y) { define <2 x i32> @vwaddu_v2i32_of_v2i8(ptr %x, ptr %y) { ; CHECK-LABEL: vwaddu_v2i32_of_v2i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetivli zero, 2, e16, mf4, ta, ma +; CHECK-NEXT: vsetivli zero, 2, e8, mf8, ta, ma ; CHECK-NEXT: vle8.v v8, (a0) ; CHECK-NEXT: vle8.v v9, (a1) -; CHECK-NEXT: vzext.vf2 v10, v8 -; CHECK-NEXT: vzext.vf2 v11, v9 -; CHECK-NEXT: vwaddu.vv v8, v10, v11 +; CHECK-NEXT: vwaddu.vv v10, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf2 v8, v10 ; CHECK-NEXT: ret %a = load <2 x i8>, ptr %x %b = load <2 x i8>, ptr %y @@ -930,12 +930,12 @@ define <2 x i32> @vwaddu_v2i32_of_v2i8(ptr %x, ptr %y) { define <2 x i64> @vwaddu_v2i64_of_v2i8(ptr %x, ptr %y) { ; CHECK-LABEL: vwaddu_v2i64_of_v2i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; CHECK-NEXT: vsetivli zero, 2, e8, mf8, ta, ma ; CHECK-NEXT: vle8.v v8, (a0) ; CHECK-NEXT: vle8.v v9, (a1) -; CHECK-NEXT: vzext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v11, v9 -; CHECK-NEXT: vwaddu.vv v8, v10, v11 +; CHECK-NEXT: vwaddu.vv v10, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma +; CHECK-NEXT: vzext.vf4 v8, v10 ; CHECK-NEXT: ret %a = load <2 x i8>, ptr %x %b = load <2 x i8>, ptr %y @@ -948,12 +948,12 @@ define <2 x i64> @vwaddu_v2i64_of_v2i8(ptr %x, ptr %y) { define <2 x i64> @vwaddu_v2i64_of_v2i16(ptr %x, ptr %y) { ; CHECK-LABEL: vwaddu_v2i64_of_v2i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; CHECK-NEXT: vsetivli zero, 2, e16, mf4, ta, ma ; CHECK-NEXT: vle16.v v8, (a0) ; CHECK-NEXT: vle16.v v9, (a1) -; CHECK-NEXT: vzext.vf2 v10, v8 -; CHECK-NEXT: vzext.vf2 v11, v9 -; CHECK-NEXT: vwaddu.vv v8, v10, v11 +; CHECK-NEXT: vwaddu.vv v10, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma +; CHECK-NEXT: vzext.vf2 v8, v10 ; CHECK-NEXT: ret %a = load <2 x i16>, ptr %x %b = load <2 x i16>, ptr %y diff --git a/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll index 66a7eea18be5..0a7051633a19 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll @@ -435,10 +435,10 @@ define @vwadd_vv_nxv1i64_nxv1i16( %va, @vwaddu_vv_nxv1i64_nxv1i16( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv1i64_nxv1i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma -; CHECK-NEXT: vzext.vf2 v10, v8 -; CHECK-NEXT: vzext.vf2 v11, v9 -; CHECK-NEXT: vwaddu.vv v8, v10, v11 +; CHECK-NEXT: vsetvli a0, zero, e16, mf4, ta, ma +; CHECK-NEXT: vwaddu.vv v10, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma +; CHECK-NEXT: vzext.vf2 v8, v10 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -468,11 +468,9 @@ define @vwaddu_vx_nxv1i64_nxv1i16( %va, i16 ; CHECK-LABEL: vwaddu_vx_nxv1i64_nxv1i16: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf4, ta, ma -; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma -; CHECK-NEXT: vzext.vf2 v10, v8 -; CHECK-NEXT: vzext.vf2 v11, v9 -; CHECK-NEXT: vwaddu.vv v8, v10, v11 +; CHECK-NEXT: vwaddu.vx v9, v8, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma +; CHECK-NEXT: vzext.vf2 v8, v9 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -555,10 +553,10 @@ define @vwadd_vv_nxv2i64_nxv2i16( %va, @vwaddu_vv_nxv2i64_nxv2i16( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv2i64_nxv2i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma -; CHECK-NEXT: vzext.vf2 v10, v8 -; CHECK-NEXT: vzext.vf2 v11, v9 -; CHECK-NEXT: vwaddu.vv v8, v10, v11 +; CHECK-NEXT: vsetvli a0, zero, e16, mf2, ta, ma +; CHECK-NEXT: vwaddu.vv v10, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v8, v10 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -588,11 +586,9 @@ define @vwaddu_vx_nxv2i64_nxv2i16( %va, i16 ; CHECK-LABEL: vwaddu_vx_nxv2i64_nxv2i16: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf2, ta, ma -; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma -; CHECK-NEXT: vzext.vf2 v10, v8 -; CHECK-NEXT: vzext.vf2 v11, v9 -; CHECK-NEXT: vwaddu.vv v8, v10, v11 +; CHECK-NEXT: vwaddu.vx v10, v8, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v8, v10 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -675,10 +671,10 @@ define @vwadd_vv_nxv4i64_nxv4i16( %va, @vwaddu_vv_nxv4i64_nxv4i16( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv4i64_nxv4i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma -; CHECK-NEXT: vzext.vf2 v12, v8 -; CHECK-NEXT: vzext.vf2 v14, v9 -; CHECK-NEXT: vwaddu.vv v8, v12, v14 +; CHECK-NEXT: vsetvli a0, zero, e16, m1, ta, ma +; CHECK-NEXT: vwaddu.vv v12, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma +; CHECK-NEXT: vzext.vf2 v8, v12 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -708,11 +704,9 @@ define @vwaddu_vx_nxv4i64_nxv4i16( %va, i16 ; CHECK-LABEL: vwaddu_vx_nxv4i64_nxv4i16: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma -; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma -; CHECK-NEXT: vzext.vf2 v12, v8 -; CHECK-NEXT: vzext.vf2 v14, v9 -; CHECK-NEXT: vwaddu.vv v8, v12, v14 +; CHECK-NEXT: vwaddu.vx v12, v8, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma +; CHECK-NEXT: vzext.vf2 v8, v12 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -795,10 +789,10 @@ define @vwadd_vv_nxv8i64_nxv8i16( %va, @vwaddu_vv_nxv8i64_nxv8i16( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv8i64_nxv8i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma -; CHECK-NEXT: vzext.vf2 v16, v8 -; CHECK-NEXT: vzext.vf2 v20, v10 -; CHECK-NEXT: vwaddu.vv v8, v16, v20 +; CHECK-NEXT: vsetvli a0, zero, e16, m2, ta, ma +; CHECK-NEXT: vwaddu.vv v16, v8, v10 +; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; CHECK-NEXT: vzext.vf2 v8, v16 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -828,11 +822,9 @@ define @vwaddu_vx_nxv8i64_nxv8i16( %va, i16 ; CHECK-LABEL: vwaddu_vx_nxv8i64_nxv8i16: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma -; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma -; CHECK-NEXT: vzext.vf2 v16, v8 -; CHECK-NEXT: vzext.vf2 v20, v10 -; CHECK-NEXT: vwaddu.vv v8, v16, v20 +; CHECK-NEXT: vwaddu.vx v16, v8, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; CHECK-NEXT: vzext.vf2 v8, v16 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -915,10 +907,10 @@ define @vwadd_vv_nxv1i64_nxv1i8( %va, @vwaddu_vv_nxv1i64_nxv1i8( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv1i64_nxv1i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v11, v9 -; CHECK-NEXT: vwaddu.vv v8, v10, v11 +; CHECK-NEXT: vsetvli a0, zero, e8, mf8, ta, ma +; CHECK-NEXT: vwaddu.vv v10, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma +; CHECK-NEXT: vzext.vf4 v8, v10 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -948,11 +940,9 @@ define @vwaddu_vx_nxv1i64_nxv1i8( %va, i8 %b ; CHECK-LABEL: vwaddu_vx_nxv1i64_nxv1i8: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma -; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v11, v9 -; CHECK-NEXT: vwaddu.vv v8, v10, v11 +; CHECK-NEXT: vwaddu.vx v9, v8, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma +; CHECK-NEXT: vzext.vf4 v8, v9 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1035,10 +1025,10 @@ define @vwadd_vv_nxv2i64_nxv2i8( %va, @vwaddu_vv_nxv2i64_nxv2i8( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv2i64_nxv2i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v11, v9 -; CHECK-NEXT: vwaddu.vv v8, v10, v11 +; CHECK-NEXT: vsetvli a0, zero, e8, mf4, ta, ma +; CHECK-NEXT: vwaddu.vv v10, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf4 v8, v10 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -1068,11 +1058,9 @@ define @vwaddu_vx_nxv2i64_nxv2i8( %va, i8 %b ; CHECK-LABEL: vwaddu_vx_nxv2i64_nxv2i8: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma -; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v11, v9 -; CHECK-NEXT: vwaddu.vv v8, v10, v11 +; CHECK-NEXT: vwaddu.vx v10, v8, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf4 v8, v10 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1155,10 +1143,10 @@ define @vwadd_vv_nxv4i64_nxv4i8( %va, @vwaddu_vv_nxv4i64_nxv4i8( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv4i64_nxv4i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma -; CHECK-NEXT: vzext.vf4 v12, v8 -; CHECK-NEXT: vzext.vf4 v14, v9 -; CHECK-NEXT: vwaddu.vv v8, v12, v14 +; CHECK-NEXT: vsetvli a0, zero, e8, mf2, ta, ma +; CHECK-NEXT: vwaddu.vv v12, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma +; CHECK-NEXT: vzext.vf4 v8, v12 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -1188,11 +1176,9 @@ define @vwaddu_vx_nxv4i64_nxv4i8( %va, i8 %b ; CHECK-LABEL: vwaddu_vx_nxv4i64_nxv4i8: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma -; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma -; CHECK-NEXT: vzext.vf4 v12, v8 -; CHECK-NEXT: vzext.vf4 v14, v9 -; CHECK-NEXT: vwaddu.vv v8, v12, v14 +; CHECK-NEXT: vwaddu.vx v12, v8, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma +; CHECK-NEXT: vzext.vf4 v8, v12 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1275,10 +1261,10 @@ define @vwadd_vv_nxv8i64_nxv8i8( %va, @vwaddu_vv_nxv8i64_nxv8i8( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv8i64_nxv8i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma -; CHECK-NEXT: vzext.vf4 v16, v8 -; CHECK-NEXT: vzext.vf4 v20, v9 -; CHECK-NEXT: vwaddu.vv v8, v16, v20 +; CHECK-NEXT: vsetvli a0, zero, e8, m1, ta, ma +; CHECK-NEXT: vwaddu.vv v16, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; CHECK-NEXT: vzext.vf4 v8, v16 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -1308,11 +1294,9 @@ define @vwaddu_vx_nxv8i64_nxv8i8( %va, i8 %b ; CHECK-LABEL: vwaddu_vx_nxv8i64_nxv8i8: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma -; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma -; CHECK-NEXT: vzext.vf4 v16, v8 -; CHECK-NEXT: vzext.vf4 v20, v9 -; CHECK-NEXT: vwaddu.vv v8, v16, v20 +; CHECK-NEXT: vwaddu.vx v16, v8, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; CHECK-NEXT: vzext.vf4 v8, v16 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer -- GitLab From babbdad15b8049a6a78087d15a163d897f07d320 Mon Sep 17 00:00:00 2001 From: Pierre van Houtryve Date: Mon, 25 Mar 2024 09:23:40 +0100 Subject: [PATCH 185/306] [AMDGPU] Handle non-register operands for S_SUB/ADD_U64_PSEUDO (#86104) This pseudo uses SSrc_b64 so it allows both an immediate or a register, but the lowering crashed on immediate operands. --- llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 4 +- .../CodeGen/AMDGPU/add_sub_u64_pseudos.mir | 68 +++++++++++++++++++ 2 files changed, 70 insertions(+), 2 deletions(-) create mode 100644 llvm/test/CodeGen/AMDGPU/add_sub_u64_pseudos.mir diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index 7f0cff72c186..d437f339a687 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -4859,8 +4859,8 @@ MachineBasicBlock *SITargetLowering::EmitInstrWithCustomInserter( if (Subtarget->hasScalarAddSub64()) { unsigned Opc = IsAdd ? AMDGPU::S_ADD_U64 : AMDGPU::S_SUB_U64; BuildMI(*BB, MI, DL, TII->get(Opc), Dest.getReg()) - .addReg(Src0.getReg()) - .addReg(Src1.getReg()); + .add(Src0) + .add(Src1); } else { const SIRegisterInfo *TRI = ST.getRegisterInfo(); const TargetRegisterClass *BoolRC = TRI->getBoolRC(); diff --git a/llvm/test/CodeGen/AMDGPU/add_sub_u64_pseudos.mir b/llvm/test/CodeGen/AMDGPU/add_sub_u64_pseudos.mir new file mode 100644 index 000000000000..cba114c3568a --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/add_sub_u64_pseudos.mir @@ -0,0 +1,68 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -run-pass=finalize-isel -o - %s | FileCheck -check-prefix=GFX11 %s +# RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -run-pass=finalize-isel -o - %s | FileCheck -check-prefix=GFX12 %s + +--- +name: reg_ops +tracksRegLiveness: true +body: | + bb.0: + ; GFX11-LABEL: name: reg_ops + ; GFX11: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF + ; GFX11-NEXT: [[DEF1:%[0-9]+]]:sreg_64 = IMPLICIT_DEF + ; GFX11-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY [[DEF]].sub0 + ; GFX11-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY [[DEF]].sub1 + ; GFX11-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY [[DEF1]].sub0 + ; GFX11-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY [[DEF1]].sub1 + ; GFX11-NEXT: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 [[COPY]], [[COPY2]], implicit-def $scc + ; GFX11-NEXT: [[S_ADDC_U32_:%[0-9]+]]:sreg_32 = S_ADDC_U32 [[COPY1]], [[COPY3]], implicit-def $scc, implicit $scc + ; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_ADD_U32_]], %subreg.sub0, [[S_ADDC_U32_]], %subreg.sub1 + ; + ; GFX12-LABEL: name: reg_ops + ; GFX12: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF + ; GFX12-NEXT: [[DEF1:%[0-9]+]]:sreg_64 = IMPLICIT_DEF + ; GFX12-NEXT: [[S_ADD_U64_:%[0-9]+]]:sreg_64 = S_ADD_U64 [[DEF]], [[DEF1]] + %0:sreg_64 = IMPLICIT_DEF + %1:sreg_64 = IMPLICIT_DEF + %2:sreg_64 = S_ADD_U64_PSEUDO %0, %1, implicit-def $scc +... + +--- +name: lhs_imm +tracksRegLiveness: true +body: | + bb.0: + ; GFX11-LABEL: name: lhs_imm + ; GFX11: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF + ; GFX11-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY [[DEF]].sub0 + ; GFX11-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY [[DEF]].sub1 + ; GFX11-NEXT: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 6565, [[COPY]], implicit-def $scc + ; GFX11-NEXT: [[S_ADDC_U32_:%[0-9]+]]:sreg_32 = S_ADDC_U32 0, [[COPY1]], implicit-def $scc, implicit $scc + ; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_ADD_U32_]], %subreg.sub0, [[S_ADDC_U32_]], %subreg.sub1 + ; + ; GFX12-LABEL: name: lhs_imm + ; GFX12: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF + ; GFX12-NEXT: [[S_ADD_U64_:%[0-9]+]]:sreg_64 = S_ADD_U64 6565, [[DEF]] + %0:sreg_64 = IMPLICIT_DEF + %1:sreg_64 = S_ADD_U64_PSEUDO 6565, %0, implicit-def $scc +... + +--- +name: rhs_imm +tracksRegLiveness: true +body: | + bb.0: + ; GFX11-LABEL: name: rhs_imm + ; GFX11: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF + ; GFX11-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY [[DEF]].sub0 + ; GFX11-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY [[DEF]].sub1 + ; GFX11-NEXT: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 [[COPY]], 6565, implicit-def $scc + ; GFX11-NEXT: [[S_ADDC_U32_:%[0-9]+]]:sreg_32 = S_ADDC_U32 [[COPY1]], 0, implicit-def $scc, implicit $scc + ; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_ADD_U32_]], %subreg.sub0, [[S_ADDC_U32_]], %subreg.sub1 + ; + ; GFX12-LABEL: name: rhs_imm + ; GFX12: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF + ; GFX12-NEXT: [[S_ADD_U64_:%[0-9]+]]:sreg_64 = S_ADD_U64 [[DEF]], 6565 + %0:sreg_64 = IMPLICIT_DEF + %1:sreg_64 = S_ADD_U64_PSEUDO %0, 6565, implicit-def $scc +... -- GitLab From fa3d789df15bd1f58fb8ba4ea3be909218cf7f03 Mon Sep 17 00:00:00 2001 From: Pierre van Houtryve Date: Mon, 25 Mar 2024 09:40:35 +0100 Subject: [PATCH 186/306] [RFC][TableGen] Restructure TableGen Source (#80847) Refactor of the llvm-tblgen source into: - a "Basic" library, which contains the bare minimum utilities to build `llvm-min-tablegen` - a "Common" library which contains all of the helpers for TableGen backends. Such helpers can be shared by more than one backend, and even unit tested (e.g. CodeExpander is, maybe we can add more over time) Fixes #80647 --- llvm/unittests/TableGen/CMakeLists.txt | 2 +- llvm/unittests/TableGen/CodeExpanderTest.cpp | 4 +- llvm/utils/TableGen/AsmMatcherEmitter.cpp | 12 ++-- llvm/utils/TableGen/AsmWriterEmitter.cpp | 14 ++-- llvm/utils/TableGen/Basic/CMakeLists.txt | 21 ++++++ .../{ => Basic}/CodeGenIntrinsics.cpp | 0 .../TableGen/{ => Basic}/CodeGenIntrinsics.h | 0 .../TableGen/{ => Basic}/SDNodeProperties.cpp | 0 .../TableGen/{ => Basic}/SDNodeProperties.h | 0 .../{ => Basic}/SequenceToOffsetTable.h | 0 llvm/utils/TableGen/CMakeLists.txt | 67 ++++++++----------- llvm/utils/TableGen/CallingConvEmitter.cpp | 2 +- llvm/utils/TableGen/CodeEmitterGen.cpp | 10 +-- llvm/utils/TableGen/CodeGenMapTable.cpp | 4 +- .../TableGen/{ => Common}/AsmWriterInst.cpp | 0 .../TableGen/{ => Common}/AsmWriterInst.h | 0 llvm/utils/TableGen/Common/CMakeLists.txt | 48 +++++++++++++ .../{ => Common}/CodeGenDAGPatterns.cpp | 0 .../{ => Common}/CodeGenDAGPatterns.h | 4 +- .../TableGen/{ => Common}/CodeGenHwModes.cpp | 0 .../TableGen/{ => Common}/CodeGenHwModes.h | 0 .../{ => Common}/CodeGenInstAlias.cpp | 0 .../TableGen/{ => Common}/CodeGenInstAlias.h | 0 .../{ => Common}/CodeGenInstruction.cpp | 0 .../{ => Common}/CodeGenInstruction.h | 0 .../{ => Common}/CodeGenRegisters.cpp | 0 .../TableGen/{ => Common}/CodeGenRegisters.h | 0 .../TableGen/{ => Common}/CodeGenSchedule.cpp | 0 .../TableGen/{ => Common}/CodeGenSchedule.h | 0 .../TableGen/{ => Common}/CodeGenTarget.cpp | 0 .../TableGen/{ => Common}/CodeGenTarget.h | 2 +- .../TableGen/{ => Common}/DAGISelMatcher.cpp | 0 .../TableGen/{ => Common}/DAGISelMatcher.h | 0 .../{ => Common}/GlobalISel/CXXPredicates.cpp | 0 .../{ => Common}/GlobalISel/CXXPredicates.h | 0 .../{ => Common}/GlobalISel/CodeExpander.cpp | 0 .../{ => Common}/GlobalISel/CodeExpander.h | 0 .../{ => Common}/GlobalISel/CodeExpansions.h | 0 .../{ => Common}/GlobalISel/CombinerUtils.h | 0 .../GlobalISel}/GlobalISelMatchTable.cpp | 4 +- .../GlobalISel}/GlobalISelMatchTable.h | 2 +- .../GlobalISelMatchTableExecutorEmitter.cpp | 0 .../GlobalISelMatchTableExecutorEmitter.h | 2 +- .../{ => Common}/GlobalISel/MatchDataInfo.cpp | 0 .../{ => Common}/GlobalISel/MatchDataInfo.h | 0 .../{ => Common}/GlobalISel/Patterns.cpp | 4 +- .../{ => Common}/GlobalISel/Patterns.h | 0 .../TableGen/{ => Common}/InfoByHwMode.cpp | 0 .../TableGen/{ => Common}/InfoByHwMode.h | 0 .../TableGen/{ => Common}/OptEmitter.cpp | 0 llvm/utils/TableGen/{ => Common}/OptEmitter.h | 0 .../{ => Common}/PredicateExpander.cpp | 0 .../TableGen/{ => Common}/PredicateExpander.h | 0 .../{ => Common}/SubtargetFeatureInfo.cpp | 0 .../{ => Common}/SubtargetFeatureInfo.h | 0 llvm/utils/TableGen/{ => Common}/Types.cpp | 0 llvm/utils/TableGen/{ => Common}/Types.h | 0 .../{ => Common}/VarLenCodeEmitterGen.cpp | 15 ++--- .../{ => Common}/VarLenCodeEmitterGen.h | 0 llvm/utils/TableGen/CompressInstEmitter.cpp | 6 +- llvm/utils/TableGen/DAGISelEmitter.cpp | 8 +-- llvm/utils/TableGen/DAGISelMatcherEmitter.cpp | 12 ++-- llvm/utils/TableGen/DAGISelMatcherGen.cpp | 14 ++-- llvm/utils/TableGen/DAGISelMatcherOpt.cpp | 6 +- llvm/utils/TableGen/DFAEmitter.cpp | 2 +- llvm/utils/TableGen/DFAPacketizerEmitter.cpp | 4 +- llvm/utils/TableGen/DXILEmitter.cpp | 4 +- llvm/utils/TableGen/DecoderEmitter.cpp | 10 +-- llvm/utils/TableGen/DisassemblerEmitter.cpp | 2 +- llvm/utils/TableGen/FastISelEmitter.cpp | 10 +-- llvm/utils/TableGen/GlobalISel/CMakeLists.txt | 20 ------ .../TableGen/GlobalISelCombinerEmitter.cpp | 24 +++---- llvm/utils/TableGen/GlobalISelEmitter.cpp | 18 ++--- llvm/utils/TableGen/InstrDocsEmitter.cpp | 6 +- llvm/utils/TableGen/InstrInfoEmitter.cpp | 16 ++--- llvm/utils/TableGen/IntrinsicEmitter.cpp | 4 +- .../TableGen/MacroFusionPredicatorEmitter.cpp | 4 +- llvm/utils/TableGen/OptParserEmitter.cpp | 2 +- llvm/utils/TableGen/OptRSTEmitter.cpp | 2 +- llvm/utils/TableGen/PseudoLoweringEmitter.cpp | 4 +- llvm/utils/TableGen/RegisterBankEmitter.cpp | 6 +- llvm/utils/TableGen/RegisterInfoEmitter.cpp | 12 ++-- .../utils/TableGen/SearchableTableEmitter.cpp | 4 +- llvm/utils/TableGen/SubtargetEmitter.cpp | 8 +-- .../WebAssemblyDisassemblerEmitter.cpp | 2 +- .../TableGen/X86CompressEVEXTablesEmitter.cpp | 4 +- llvm/utils/TableGen/X86FoldTablesEmitter.cpp | 4 +- llvm/utils/TableGen/X86MnemonicTables.cpp | 4 +- llvm/utils/TableGen/X86RecognizableInstr.h | 2 +- 89 files changed, 238 insertions(+), 203 deletions(-) create mode 100644 llvm/utils/TableGen/Basic/CMakeLists.txt rename llvm/utils/TableGen/{ => Basic}/CodeGenIntrinsics.cpp (100%) rename llvm/utils/TableGen/{ => Basic}/CodeGenIntrinsics.h (100%) rename llvm/utils/TableGen/{ => Basic}/SDNodeProperties.cpp (100%) rename llvm/utils/TableGen/{ => Basic}/SDNodeProperties.h (100%) rename llvm/utils/TableGen/{ => Basic}/SequenceToOffsetTable.h (100%) rename llvm/utils/TableGen/{ => Common}/AsmWriterInst.cpp (100%) rename llvm/utils/TableGen/{ => Common}/AsmWriterInst.h (100%) create mode 100644 llvm/utils/TableGen/Common/CMakeLists.txt rename llvm/utils/TableGen/{ => Common}/CodeGenDAGPatterns.cpp (100%) rename llvm/utils/TableGen/{ => Common}/CodeGenDAGPatterns.h (99%) rename llvm/utils/TableGen/{ => Common}/CodeGenHwModes.cpp (100%) rename llvm/utils/TableGen/{ => Common}/CodeGenHwModes.h (100%) rename llvm/utils/TableGen/{ => Common}/CodeGenInstAlias.cpp (100%) rename llvm/utils/TableGen/{ => Common}/CodeGenInstAlias.h (100%) rename llvm/utils/TableGen/{ => Common}/CodeGenInstruction.cpp (100%) rename llvm/utils/TableGen/{ => Common}/CodeGenInstruction.h (100%) rename llvm/utils/TableGen/{ => Common}/CodeGenRegisters.cpp (100%) rename llvm/utils/TableGen/{ => Common}/CodeGenRegisters.h (100%) rename llvm/utils/TableGen/{ => Common}/CodeGenSchedule.cpp (100%) rename llvm/utils/TableGen/{ => Common}/CodeGenSchedule.h (100%) rename llvm/utils/TableGen/{ => Common}/CodeGenTarget.cpp (100%) rename llvm/utils/TableGen/{ => Common}/CodeGenTarget.h (99%) rename llvm/utils/TableGen/{ => Common}/DAGISelMatcher.cpp (100%) rename llvm/utils/TableGen/{ => Common}/DAGISelMatcher.h (100%) rename llvm/utils/TableGen/{ => Common}/GlobalISel/CXXPredicates.cpp (100%) rename llvm/utils/TableGen/{ => Common}/GlobalISel/CXXPredicates.h (100%) rename llvm/utils/TableGen/{ => Common}/GlobalISel/CodeExpander.cpp (100%) rename llvm/utils/TableGen/{ => Common}/GlobalISel/CodeExpander.h (100%) rename llvm/utils/TableGen/{ => Common}/GlobalISel/CodeExpansions.h (100%) rename llvm/utils/TableGen/{ => Common}/GlobalISel/CombinerUtils.h (100%) rename llvm/utils/TableGen/{ => Common/GlobalISel}/GlobalISelMatchTable.cpp (99%) rename llvm/utils/TableGen/{ => Common/GlobalISel}/GlobalISelMatchTable.h (99%) rename llvm/utils/TableGen/{ => Common/GlobalISel}/GlobalISelMatchTableExecutorEmitter.cpp (100%) rename llvm/utils/TableGen/{ => Common/GlobalISel}/GlobalISelMatchTableExecutorEmitter.h (99%) rename llvm/utils/TableGen/{ => Common}/GlobalISel/MatchDataInfo.cpp (100%) rename llvm/utils/TableGen/{ => Common}/GlobalISel/MatchDataInfo.h (100%) rename llvm/utils/TableGen/{ => Common}/GlobalISel/Patterns.cpp (99%) rename llvm/utils/TableGen/{ => Common}/GlobalISel/Patterns.h (100%) rename llvm/utils/TableGen/{ => Common}/InfoByHwMode.cpp (100%) rename llvm/utils/TableGen/{ => Common}/InfoByHwMode.h (100%) rename llvm/utils/TableGen/{ => Common}/OptEmitter.cpp (100%) rename llvm/utils/TableGen/{ => Common}/OptEmitter.h (100%) rename llvm/utils/TableGen/{ => Common}/PredicateExpander.cpp (100%) rename llvm/utils/TableGen/{ => Common}/PredicateExpander.h (100%) rename llvm/utils/TableGen/{ => Common}/SubtargetFeatureInfo.cpp (100%) rename llvm/utils/TableGen/{ => Common}/SubtargetFeatureInfo.h (100%) rename llvm/utils/TableGen/{ => Common}/Types.cpp (100%) rename llvm/utils/TableGen/{ => Common}/Types.h (100%) rename llvm/utils/TableGen/{ => Common}/VarLenCodeEmitterGen.cpp (97%) rename llvm/utils/TableGen/{ => Common}/VarLenCodeEmitterGen.h (100%) delete mode 100644 llvm/utils/TableGen/GlobalISel/CMakeLists.txt diff --git a/llvm/unittests/TableGen/CMakeLists.txt b/llvm/unittests/TableGen/CMakeLists.txt index 7830e0218045..fae0eee06e4b 100644 --- a/llvm/unittests/TableGen/CMakeLists.txt +++ b/llvm/unittests/TableGen/CMakeLists.txt @@ -15,4 +15,4 @@ add_llvm_unittest(TableGenTests DISABLE_LLVM_LINK_LLVM_DYLIB ParserEntryPointTest.cpp ) -target_link_libraries(TableGenTests PRIVATE LLVMTableGenGlobalISel LLVMTableGen) +target_link_libraries(TableGenTests PRIVATE LLVMTableGenCommon LLVMTableGen) diff --git a/llvm/unittests/TableGen/CodeExpanderTest.cpp b/llvm/unittests/TableGen/CodeExpanderTest.cpp index 4a9a0e8c114b..1528884ffdf6 100644 --- a/llvm/unittests/TableGen/CodeExpanderTest.cpp +++ b/llvm/unittests/TableGen/CodeExpanderTest.cpp @@ -6,8 +6,8 @@ // //===----------------------------------------------------------------------===// -#include "GlobalISel/CodeExpander.h" -#include "GlobalISel/CodeExpansions.h" +#include "Common/GlobalISel/CodeExpander.h" +#include "Common/GlobalISel/CodeExpansions.h" #include "llvm/Support/raw_ostream.h" #include "llvm/TableGen/Error.h" diff --git a/llvm/utils/TableGen/AsmMatcherEmitter.cpp b/llvm/utils/TableGen/AsmMatcherEmitter.cpp index 0a2b1cf0f9a6..8b82ce899a48 100644 --- a/llvm/utils/TableGen/AsmMatcherEmitter.cpp +++ b/llvm/utils/TableGen/AsmMatcherEmitter.cpp @@ -95,12 +95,12 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenInstAlias.h" -#include "CodeGenInstruction.h" -#include "CodeGenRegisters.h" -#include "CodeGenTarget.h" -#include "SubtargetFeatureInfo.h" -#include "Types.h" +#include "Common/CodeGenInstAlias.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenRegisters.h" +#include "Common/CodeGenTarget.h" +#include "Common/SubtargetFeatureInfo.h" +#include "Common/Types.h" #include "llvm/ADT/CachedHashString.h" #include "llvm/ADT/PointerUnion.h" #include "llvm/ADT/STLExtras.h" diff --git a/llvm/utils/TableGen/AsmWriterEmitter.cpp b/llvm/utils/TableGen/AsmWriterEmitter.cpp index a27061ee585a..16661cd29edc 100644 --- a/llvm/utils/TableGen/AsmWriterEmitter.cpp +++ b/llvm/utils/TableGen/AsmWriterEmitter.cpp @@ -11,13 +11,13 @@ // //===----------------------------------------------------------------------===// -#include "AsmWriterInst.h" -#include "CodeGenInstAlias.h" -#include "CodeGenInstruction.h" -#include "CodeGenRegisters.h" -#include "CodeGenTarget.h" -#include "SequenceToOffsetTable.h" -#include "Types.h" +#include "Basic/SequenceToOffsetTable.h" +#include "Common/AsmWriterInst.h" +#include "Common/CodeGenInstAlias.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenRegisters.h" +#include "Common/CodeGenTarget.h" +#include "Common/Types.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/STLExtras.h" diff --git a/llvm/utils/TableGen/Basic/CMakeLists.txt b/llvm/utils/TableGen/Basic/CMakeLists.txt new file mode 100644 index 000000000000..f2927d05c175 --- /dev/null +++ b/llvm/utils/TableGen/Basic/CMakeLists.txt @@ -0,0 +1,21 @@ +# The basic TableGen library contains as little dependencies as possible. +# In particular, it does not depend on vt_gen -> it does not use ValueTypes. +# +# This library is the only thing included in `llvm-min-tablegen`. + +set(LLVM_LINK_COMPONENTS + Support + TableGen + ) + +add_llvm_library(LLVMTableGenBasic STATIC OBJECT EXCLUDE_FROM_ALL + CodeGenIntrinsics.cpp + SDNodeProperties.cpp +) +set_target_properties(LLVMTableGenBasic PROPERTIES FOLDER "Tablegenning") + +# Users may include its headers as "Basic/*.h" +target_include_directories(LLVMTableGenBasic + INTERFACE + $ + ) diff --git a/llvm/utils/TableGen/CodeGenIntrinsics.cpp b/llvm/utils/TableGen/Basic/CodeGenIntrinsics.cpp similarity index 100% rename from llvm/utils/TableGen/CodeGenIntrinsics.cpp rename to llvm/utils/TableGen/Basic/CodeGenIntrinsics.cpp diff --git a/llvm/utils/TableGen/CodeGenIntrinsics.h b/llvm/utils/TableGen/Basic/CodeGenIntrinsics.h similarity index 100% rename from llvm/utils/TableGen/CodeGenIntrinsics.h rename to llvm/utils/TableGen/Basic/CodeGenIntrinsics.h diff --git a/llvm/utils/TableGen/SDNodeProperties.cpp b/llvm/utils/TableGen/Basic/SDNodeProperties.cpp similarity index 100% rename from llvm/utils/TableGen/SDNodeProperties.cpp rename to llvm/utils/TableGen/Basic/SDNodeProperties.cpp diff --git a/llvm/utils/TableGen/SDNodeProperties.h b/llvm/utils/TableGen/Basic/SDNodeProperties.h similarity index 100% rename from llvm/utils/TableGen/SDNodeProperties.h rename to llvm/utils/TableGen/Basic/SDNodeProperties.h diff --git a/llvm/utils/TableGen/SequenceToOffsetTable.h b/llvm/utils/TableGen/Basic/SequenceToOffsetTable.h similarity index 100% rename from llvm/utils/TableGen/SequenceToOffsetTable.h rename to llvm/utils/TableGen/Basic/SequenceToOffsetTable.h diff --git a/llvm/utils/TableGen/CMakeLists.txt b/llvm/utils/TableGen/CMakeLists.txt index 0100bf345ec2..14690329cabf 100644 --- a/llvm/utils/TableGen/CMakeLists.txt +++ b/llvm/utils/TableGen/CMakeLists.txt @@ -1,26 +1,25 @@ -add_subdirectory(GlobalISel) +# Basic utilities which is the strict minimum needed to build +# llvm-min-tblgen. +add_subdirectory(Basic) +# Common utilities are all of the reusable components and helper +# code needed by the backends. +add_subdirectory(Common) -add_llvm_library(LLVMTableGenCommon STATIC OBJECT EXCLUDE_FROM_ALL +set(LLVM_LINK_COMPONENTS Support) + +# llvm-min-tablegen only contains a subset of backends necessary to +# build llvm/include. It must not depend on TableGenCommon, as +# TableGenCommon depends on this already to generate things such as +# ValueType definitions. +add_tablegen(llvm-min-tblgen LLVM_HEADERS + TableGen.cpp Attributes.cpp - CodeGenIntrinsics.cpp DirectiveEmitter.cpp IntrinsicEmitter.cpp RISCVTargetDefEmitter.cpp - SDNodeProperties.cpp VTEmitter.cpp - PARTIAL_SOURCES_INTENDED - - LINK_COMPONENTS - Support - TableGen - ) -set_target_properties(LLVMTableGenCommon PROPERTIES FOLDER "Tablegenning") + $ -set(LLVM_LINK_COMPONENTS Support) - -add_tablegen(llvm-min-tblgen LLVM_HEADERS - TableGen.cpp - $ PARTIAL_SOURCES_INTENDED ) set_target_properties(llvm-min-tblgen PROPERTIES FOLDER "Tablegenning") @@ -35,63 +34,51 @@ add_tablegen(llvm-tblgen LLVM EXPORT LLVM AsmMatcherEmitter.cpp AsmWriterEmitter.cpp - AsmWriterInst.cpp - CTagsEmitter.cpp + Attributes.cpp CallingConvEmitter.cpp CodeEmitterGen.cpp - CodeGenDAGPatterns.cpp - CodeGenHwModes.cpp - CodeGenInstAlias.cpp - CodeGenInstruction.cpp CodeGenMapTable.cpp - CodeGenRegisters.cpp - CodeGenSchedule.cpp - CodeGenTarget.cpp + CompressInstEmitter.cpp + CTagsEmitter.cpp DAGISelEmitter.cpp DAGISelMatcherEmitter.cpp DAGISelMatcherGen.cpp DAGISelMatcherOpt.cpp - DAGISelMatcher.cpp DecoderEmitter.cpp DFAEmitter.cpp DFAPacketizerEmitter.cpp + DirectiveEmitter.cpp DisassemblerEmitter.cpp DXILEmitter.cpp ExegesisEmitter.cpp FastISelEmitter.cpp GlobalISelCombinerEmitter.cpp GlobalISelEmitter.cpp - GlobalISelMatchTable.cpp - GlobalISelMatchTableExecutorEmitter.cpp - InfoByHwMode.cpp - InstrInfoEmitter.cpp InstrDocsEmitter.cpp - OptEmitter.cpp + InstrInfoEmitter.cpp + IntrinsicEmitter.cpp + MacroFusionPredicatorEmitter.cpp OptParserEmitter.cpp OptRSTEmitter.cpp - PredicateExpander.cpp PseudoLoweringEmitter.cpp - CompressInstEmitter.cpp - MacroFusionPredicatorEmitter.cpp RegisterBankEmitter.cpp RegisterInfoEmitter.cpp + RISCVTargetDefEmitter.cpp SearchableTableEmitter.cpp SubtargetEmitter.cpp - SubtargetFeatureInfo.cpp TableGen.cpp - Types.cpp - VarLenCodeEmitterGen.cpp - X86DisassemblerTables.cpp + VTEmitter.cpp + WebAssemblyDisassemblerEmitter.cpp X86CompressEVEXTablesEmitter.cpp + X86DisassemblerTables.cpp X86FoldTablesEmitter.cpp X86MnemonicTables.cpp X86ModRMFilters.cpp X86RecognizableInstr.cpp - WebAssemblyDisassemblerEmitter.cpp $ DEPENDS intrinsics_gen # via llvm-min-tablegen ) -target_link_libraries(llvm-tblgen PRIVATE LLVMTableGenGlobalISel) +target_link_libraries(llvm-tblgen PRIVATE LLVMTableGenCommon) set_target_properties(llvm-tblgen PROPERTIES FOLDER "Tablegenning") diff --git a/llvm/utils/TableGen/CallingConvEmitter.cpp b/llvm/utils/TableGen/CallingConvEmitter.cpp index 3c3a2874ce80..ec6ef56a66fa 100644 --- a/llvm/utils/TableGen/CallingConvEmitter.cpp +++ b/llvm/utils/TableGen/CallingConvEmitter.cpp @@ -11,7 +11,7 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenTarget.h" +#include "Common/CodeGenTarget.h" #include "llvm/TableGen/Error.h" #include "llvm/TableGen/Record.h" #include "llvm/TableGen/TableGenBackend.h" diff --git a/llvm/utils/TableGen/CodeEmitterGen.cpp b/llvm/utils/TableGen/CodeEmitterGen.cpp index 9194c13ccdcb..a57885f22d7e 100644 --- a/llvm/utils/TableGen/CodeEmitterGen.cpp +++ b/llvm/utils/TableGen/CodeEmitterGen.cpp @@ -22,11 +22,11 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenHwModes.h" -#include "CodeGenInstruction.h" -#include "CodeGenTarget.h" -#include "InfoByHwMode.h" -#include "VarLenCodeEmitterGen.h" +#include "Common/CodeGenHwModes.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenTarget.h" +#include "Common/InfoByHwMode.h" +#include "Common/VarLenCodeEmitterGen.h" #include "llvm/ADT/APInt.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/StringExtras.h" diff --git a/llvm/utils/TableGen/CodeGenMapTable.cpp b/llvm/utils/TableGen/CodeGenMapTable.cpp index 03af0b49ba97..fbf1d47c0327 100644 --- a/llvm/utils/TableGen/CodeGenMapTable.cpp +++ b/llvm/utils/TableGen/CodeGenMapTable.cpp @@ -75,8 +75,8 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenInstruction.h" -#include "CodeGenTarget.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenTarget.h" #include "llvm/TableGen/Error.h" #include "llvm/TableGen/Record.h" using namespace llvm; diff --git a/llvm/utils/TableGen/AsmWriterInst.cpp b/llvm/utils/TableGen/Common/AsmWriterInst.cpp similarity index 100% rename from llvm/utils/TableGen/AsmWriterInst.cpp rename to llvm/utils/TableGen/Common/AsmWriterInst.cpp diff --git a/llvm/utils/TableGen/AsmWriterInst.h b/llvm/utils/TableGen/Common/AsmWriterInst.h similarity index 100% rename from llvm/utils/TableGen/AsmWriterInst.h rename to llvm/utils/TableGen/Common/AsmWriterInst.h diff --git a/llvm/utils/TableGen/Common/CMakeLists.txt b/llvm/utils/TableGen/Common/CMakeLists.txt new file mode 100644 index 000000000000..491d9bd2949d --- /dev/null +++ b/llvm/utils/TableGen/Common/CMakeLists.txt @@ -0,0 +1,48 @@ +# The common library is similar to the basic library except it can +# depend on vt_gen. +# +# This library contains the bulk of the supporting code for all +# TableGen backends. It's split off as a separate library to +# allow unit-testing those components. + +set(LLVM_LINK_COMPONENTS + Support + TableGen + ) + +add_llvm_library(LLVMTableGenCommon STATIC OBJECT EXCLUDE_FROM_ALL + GlobalISel/CodeExpander.cpp + GlobalISel/CXXPredicates.cpp + GlobalISel/GlobalISelMatchTable.cpp + GlobalISel/GlobalISelMatchTableExecutorEmitter.cpp + GlobalISel/MatchDataInfo.cpp + GlobalISel/Patterns.cpp + + AsmWriterInst.cpp + CodeGenDAGPatterns.cpp + CodeGenHwModes.cpp + CodeGenInstAlias.cpp + CodeGenInstruction.cpp + CodeGenRegisters.cpp + CodeGenSchedule.cpp + CodeGenTarget.cpp + DAGISelMatcher.cpp + InfoByHwMode.cpp + OptEmitter.cpp + PredicateExpander.cpp + SubtargetFeatureInfo.cpp + Types.cpp + VarLenCodeEmitterGen.cpp + $ + + DEPENDS + vt_gen + ) +set_target_properties(LLVMTableGenCommon PROPERTIES FOLDER "Tablegenning") +target_link_libraries(LLVMTableGenCommon PUBLIC LLVMTableGenBasic) + +# Users may include its headers as "Common/*.h" +target_include_directories(LLVMTableGenCommon + PUBLIC + $ + ) diff --git a/llvm/utils/TableGen/CodeGenDAGPatterns.cpp b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp similarity index 100% rename from llvm/utils/TableGen/CodeGenDAGPatterns.cpp rename to llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp diff --git a/llvm/utils/TableGen/CodeGenDAGPatterns.h b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.h similarity index 99% rename from llvm/utils/TableGen/CodeGenDAGPatterns.h rename to llvm/utils/TableGen/Common/CodeGenDAGPatterns.h index 823c40c922cb..7fcd39a9e940 100644 --- a/llvm/utils/TableGen/CodeGenDAGPatterns.h +++ b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.h @@ -14,9 +14,9 @@ #ifndef LLVM_UTILS_TABLEGEN_CODEGENDAGPATTERNS_H #define LLVM_UTILS_TABLEGEN_CODEGENDAGPATTERNS_H -#include "CodeGenIntrinsics.h" +#include "Basic/CodeGenIntrinsics.h" +#include "Basic/SDNodeProperties.h" #include "CodeGenTarget.h" -#include "SDNodeProperties.h" #include "llvm/ADT/IntrusiveRefCntPtr.h" #include "llvm/ADT/MapVector.h" #include "llvm/ADT/PointerUnion.h" diff --git a/llvm/utils/TableGen/CodeGenHwModes.cpp b/llvm/utils/TableGen/Common/CodeGenHwModes.cpp similarity index 100% rename from llvm/utils/TableGen/CodeGenHwModes.cpp rename to llvm/utils/TableGen/Common/CodeGenHwModes.cpp diff --git a/llvm/utils/TableGen/CodeGenHwModes.h b/llvm/utils/TableGen/Common/CodeGenHwModes.h similarity index 100% rename from llvm/utils/TableGen/CodeGenHwModes.h rename to llvm/utils/TableGen/Common/CodeGenHwModes.h diff --git a/llvm/utils/TableGen/CodeGenInstAlias.cpp b/llvm/utils/TableGen/Common/CodeGenInstAlias.cpp similarity index 100% rename from llvm/utils/TableGen/CodeGenInstAlias.cpp rename to llvm/utils/TableGen/Common/CodeGenInstAlias.cpp diff --git a/llvm/utils/TableGen/CodeGenInstAlias.h b/llvm/utils/TableGen/Common/CodeGenInstAlias.h similarity index 100% rename from llvm/utils/TableGen/CodeGenInstAlias.h rename to llvm/utils/TableGen/Common/CodeGenInstAlias.h diff --git a/llvm/utils/TableGen/CodeGenInstruction.cpp b/llvm/utils/TableGen/Common/CodeGenInstruction.cpp similarity index 100% rename from llvm/utils/TableGen/CodeGenInstruction.cpp rename to llvm/utils/TableGen/Common/CodeGenInstruction.cpp diff --git a/llvm/utils/TableGen/CodeGenInstruction.h b/llvm/utils/TableGen/Common/CodeGenInstruction.h similarity index 100% rename from llvm/utils/TableGen/CodeGenInstruction.h rename to llvm/utils/TableGen/Common/CodeGenInstruction.h diff --git a/llvm/utils/TableGen/CodeGenRegisters.cpp b/llvm/utils/TableGen/Common/CodeGenRegisters.cpp similarity index 100% rename from llvm/utils/TableGen/CodeGenRegisters.cpp rename to llvm/utils/TableGen/Common/CodeGenRegisters.cpp diff --git a/llvm/utils/TableGen/CodeGenRegisters.h b/llvm/utils/TableGen/Common/CodeGenRegisters.h similarity index 100% rename from llvm/utils/TableGen/CodeGenRegisters.h rename to llvm/utils/TableGen/Common/CodeGenRegisters.h diff --git a/llvm/utils/TableGen/CodeGenSchedule.cpp b/llvm/utils/TableGen/Common/CodeGenSchedule.cpp similarity index 100% rename from llvm/utils/TableGen/CodeGenSchedule.cpp rename to llvm/utils/TableGen/Common/CodeGenSchedule.cpp diff --git a/llvm/utils/TableGen/CodeGenSchedule.h b/llvm/utils/TableGen/Common/CodeGenSchedule.h similarity index 100% rename from llvm/utils/TableGen/CodeGenSchedule.h rename to llvm/utils/TableGen/Common/CodeGenSchedule.h diff --git a/llvm/utils/TableGen/CodeGenTarget.cpp b/llvm/utils/TableGen/Common/CodeGenTarget.cpp similarity index 100% rename from llvm/utils/TableGen/CodeGenTarget.cpp rename to llvm/utils/TableGen/Common/CodeGenTarget.cpp diff --git a/llvm/utils/TableGen/CodeGenTarget.h b/llvm/utils/TableGen/Common/CodeGenTarget.h similarity index 99% rename from llvm/utils/TableGen/CodeGenTarget.h rename to llvm/utils/TableGen/Common/CodeGenTarget.h index e109c717dc01..df4c22ebb379 100644 --- a/llvm/utils/TableGen/CodeGenTarget.h +++ b/llvm/utils/TableGen/Common/CodeGenTarget.h @@ -16,10 +16,10 @@ #ifndef LLVM_UTILS_TABLEGEN_CODEGENTARGET_H #define LLVM_UTILS_TABLEGEN_CODEGENTARGET_H +#include "Basic/SDNodeProperties.h" #include "CodeGenHwModes.h" #include "CodeGenInstruction.h" #include "InfoByHwMode.h" -#include "SDNodeProperties.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallVector.h" diff --git a/llvm/utils/TableGen/DAGISelMatcher.cpp b/llvm/utils/TableGen/Common/DAGISelMatcher.cpp similarity index 100% rename from llvm/utils/TableGen/DAGISelMatcher.cpp rename to llvm/utils/TableGen/Common/DAGISelMatcher.cpp diff --git a/llvm/utils/TableGen/DAGISelMatcher.h b/llvm/utils/TableGen/Common/DAGISelMatcher.h similarity index 100% rename from llvm/utils/TableGen/DAGISelMatcher.h rename to llvm/utils/TableGen/Common/DAGISelMatcher.h diff --git a/llvm/utils/TableGen/GlobalISel/CXXPredicates.cpp b/llvm/utils/TableGen/Common/GlobalISel/CXXPredicates.cpp similarity index 100% rename from llvm/utils/TableGen/GlobalISel/CXXPredicates.cpp rename to llvm/utils/TableGen/Common/GlobalISel/CXXPredicates.cpp diff --git a/llvm/utils/TableGen/GlobalISel/CXXPredicates.h b/llvm/utils/TableGen/Common/GlobalISel/CXXPredicates.h similarity index 100% rename from llvm/utils/TableGen/GlobalISel/CXXPredicates.h rename to llvm/utils/TableGen/Common/GlobalISel/CXXPredicates.h diff --git a/llvm/utils/TableGen/GlobalISel/CodeExpander.cpp b/llvm/utils/TableGen/Common/GlobalISel/CodeExpander.cpp similarity index 100% rename from llvm/utils/TableGen/GlobalISel/CodeExpander.cpp rename to llvm/utils/TableGen/Common/GlobalISel/CodeExpander.cpp diff --git a/llvm/utils/TableGen/GlobalISel/CodeExpander.h b/llvm/utils/TableGen/Common/GlobalISel/CodeExpander.h similarity index 100% rename from llvm/utils/TableGen/GlobalISel/CodeExpander.h rename to llvm/utils/TableGen/Common/GlobalISel/CodeExpander.h diff --git a/llvm/utils/TableGen/GlobalISel/CodeExpansions.h b/llvm/utils/TableGen/Common/GlobalISel/CodeExpansions.h similarity index 100% rename from llvm/utils/TableGen/GlobalISel/CodeExpansions.h rename to llvm/utils/TableGen/Common/GlobalISel/CodeExpansions.h diff --git a/llvm/utils/TableGen/GlobalISel/CombinerUtils.h b/llvm/utils/TableGen/Common/GlobalISel/CombinerUtils.h similarity index 100% rename from llvm/utils/TableGen/GlobalISel/CombinerUtils.h rename to llvm/utils/TableGen/Common/GlobalISel/CombinerUtils.h diff --git a/llvm/utils/TableGen/GlobalISelMatchTable.cpp b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp similarity index 99% rename from llvm/utils/TableGen/GlobalISelMatchTable.cpp rename to llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp index 45fb41b89f27..193f95443b16 100644 --- a/llvm/utils/TableGen/GlobalISelMatchTable.cpp +++ b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp @@ -7,8 +7,8 @@ //===----------------------------------------------------------------------===// #include "GlobalISelMatchTable.h" -#include "CodeGenInstruction.h" -#include "CodeGenRegisters.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenRegisters.h" #include "llvm/ADT/Statistic.h" #include "llvm/Support/Debug.h" #include "llvm/Support/LEB128.h" diff --git a/llvm/utils/TableGen/GlobalISelMatchTable.h b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h similarity index 99% rename from llvm/utils/TableGen/GlobalISelMatchTable.h rename to llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h index b1ab7da8db44..aa86fad763d1 100644 --- a/llvm/utils/TableGen/GlobalISelMatchTable.h +++ b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h @@ -16,7 +16,7 @@ #ifndef LLVM_UTILS_TABLEGEN_GLOBALISELMATCHTABLE_H #define LLVM_UTILS_TABLEGEN_GLOBALISELMATCHTABLE_H -#include "CodeGenDAGPatterns.h" +#include "Common/CodeGenDAGPatterns.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallPtrSet.h" diff --git a/llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.cpp b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.cpp similarity index 100% rename from llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.cpp rename to llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.cpp diff --git a/llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.h b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.h similarity index 99% rename from llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.h rename to llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.h index 7e952d6df309..d2b6a74c7577 100644 --- a/llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.h +++ b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.h @@ -15,7 +15,7 @@ #ifndef LLVM_UTILS_TABLEGEN_GLOBALISELMATCHTABLEEXECUTOREMITTER_H #define LLVM_UTILS_TABLEGEN_GLOBALISELMATCHTABLEEXECUTOREMITTER_H -#include "SubtargetFeatureInfo.h" +#include "Common/SubtargetFeatureInfo.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Twine.h" diff --git a/llvm/utils/TableGen/GlobalISel/MatchDataInfo.cpp b/llvm/utils/TableGen/Common/GlobalISel/MatchDataInfo.cpp similarity index 100% rename from llvm/utils/TableGen/GlobalISel/MatchDataInfo.cpp rename to llvm/utils/TableGen/Common/GlobalISel/MatchDataInfo.cpp diff --git a/llvm/utils/TableGen/GlobalISel/MatchDataInfo.h b/llvm/utils/TableGen/Common/GlobalISel/MatchDataInfo.h similarity index 100% rename from llvm/utils/TableGen/GlobalISel/MatchDataInfo.h rename to llvm/utils/TableGen/Common/GlobalISel/MatchDataInfo.h diff --git a/llvm/utils/TableGen/GlobalISel/Patterns.cpp b/llvm/utils/TableGen/Common/GlobalISel/Patterns.cpp similarity index 99% rename from llvm/utils/TableGen/GlobalISel/Patterns.cpp rename to llvm/utils/TableGen/Common/GlobalISel/Patterns.cpp index 758eac2dfebd..388bf7e9e833 100644 --- a/llvm/utils/TableGen/GlobalISel/Patterns.cpp +++ b/llvm/utils/TableGen/Common/GlobalISel/Patterns.cpp @@ -7,11 +7,11 @@ //===----------------------------------------------------------------------===// #include "Patterns.h" -#include "../CodeGenInstruction.h" -#include "../CodeGenIntrinsics.h" +#include "Basic/CodeGenIntrinsics.h" #include "CXXPredicates.h" #include "CodeExpander.h" #include "CodeExpansions.h" +#include "Common/CodeGenInstruction.h" #include "llvm/ADT/StringSet.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" diff --git a/llvm/utils/TableGen/GlobalISel/Patterns.h b/llvm/utils/TableGen/Common/GlobalISel/Patterns.h similarity index 100% rename from llvm/utils/TableGen/GlobalISel/Patterns.h rename to llvm/utils/TableGen/Common/GlobalISel/Patterns.h diff --git a/llvm/utils/TableGen/InfoByHwMode.cpp b/llvm/utils/TableGen/Common/InfoByHwMode.cpp similarity index 100% rename from llvm/utils/TableGen/InfoByHwMode.cpp rename to llvm/utils/TableGen/Common/InfoByHwMode.cpp diff --git a/llvm/utils/TableGen/InfoByHwMode.h b/llvm/utils/TableGen/Common/InfoByHwMode.h similarity index 100% rename from llvm/utils/TableGen/InfoByHwMode.h rename to llvm/utils/TableGen/Common/InfoByHwMode.h diff --git a/llvm/utils/TableGen/OptEmitter.cpp b/llvm/utils/TableGen/Common/OptEmitter.cpp similarity index 100% rename from llvm/utils/TableGen/OptEmitter.cpp rename to llvm/utils/TableGen/Common/OptEmitter.cpp diff --git a/llvm/utils/TableGen/OptEmitter.h b/llvm/utils/TableGen/Common/OptEmitter.h similarity index 100% rename from llvm/utils/TableGen/OptEmitter.h rename to llvm/utils/TableGen/Common/OptEmitter.h diff --git a/llvm/utils/TableGen/PredicateExpander.cpp b/llvm/utils/TableGen/Common/PredicateExpander.cpp similarity index 100% rename from llvm/utils/TableGen/PredicateExpander.cpp rename to llvm/utils/TableGen/Common/PredicateExpander.cpp diff --git a/llvm/utils/TableGen/PredicateExpander.h b/llvm/utils/TableGen/Common/PredicateExpander.h similarity index 100% rename from llvm/utils/TableGen/PredicateExpander.h rename to llvm/utils/TableGen/Common/PredicateExpander.h diff --git a/llvm/utils/TableGen/SubtargetFeatureInfo.cpp b/llvm/utils/TableGen/Common/SubtargetFeatureInfo.cpp similarity index 100% rename from llvm/utils/TableGen/SubtargetFeatureInfo.cpp rename to llvm/utils/TableGen/Common/SubtargetFeatureInfo.cpp diff --git a/llvm/utils/TableGen/SubtargetFeatureInfo.h b/llvm/utils/TableGen/Common/SubtargetFeatureInfo.h similarity index 100% rename from llvm/utils/TableGen/SubtargetFeatureInfo.h rename to llvm/utils/TableGen/Common/SubtargetFeatureInfo.h diff --git a/llvm/utils/TableGen/Types.cpp b/llvm/utils/TableGen/Common/Types.cpp similarity index 100% rename from llvm/utils/TableGen/Types.cpp rename to llvm/utils/TableGen/Common/Types.cpp diff --git a/llvm/utils/TableGen/Types.h b/llvm/utils/TableGen/Common/Types.h similarity index 100% rename from llvm/utils/TableGen/Types.h rename to llvm/utils/TableGen/Common/Types.h diff --git a/llvm/utils/TableGen/VarLenCodeEmitterGen.cpp b/llvm/utils/TableGen/Common/VarLenCodeEmitterGen.cpp similarity index 97% rename from llvm/utils/TableGen/VarLenCodeEmitterGen.cpp rename to llvm/utils/TableGen/Common/VarLenCodeEmitterGen.cpp index bfb7e5c33317..4263d8f41715 100644 --- a/llvm/utils/TableGen/VarLenCodeEmitterGen.cpp +++ b/llvm/utils/TableGen/Common/VarLenCodeEmitterGen.cpp @@ -337,8 +337,8 @@ static void emitInstBits(raw_ostream &IS, raw_ostream &SS, const APInt &Bits, return; } - IS.indent(4) << "{/*NumBits*/" << Bits.getBitWidth() << ", " - << "/*Index*/" << Index << "},"; + IS.indent(4) << "{/*NumBits*/" << Bits.getBitWidth() << ", " << "/*Index*/" + << Index << "},"; SS.indent(4); for (unsigned I = 0; I < Bits.getNumWords(); ++I, ++Index) @@ -371,8 +371,8 @@ void VarLenCodeEmitterGen::emitInstructionBaseValues( if (ModeIt == InstIt->second.end()) ModeIt = InstIt->second.find(Universal); if (ModeIt == InstIt->second.end()) { - IS.indent(4) << "{/*NumBits*/0, /*Index*/0},\t" - << "// " << R->getName() << " no encoding\n"; + IS.indent(4) << "{/*NumBits*/0, /*Index*/0},\t" << "// " << R->getName() + << " no encoding\n"; continue; } const VarLenInst &VLI = ModeIt->second; @@ -492,10 +492,9 @@ std::string VarLenCodeEmitterGen::getInstructionCaseForEncoding( SS << ", /*Pos=*/" << utostr(Offset) << ", Scratch, Fixups, STI);\n"; - SS.indent(I) << "Inst.insertBits(" - << "Scratch.extractBits(" << utostr(NumBits) << ", " - << utostr(LoBit) << ")" - << ", " << Offset << ");\n"; + SS.indent(I) << "Inst.insertBits(" << "Scratch.extractBits(" + << utostr(NumBits) << ", " << utostr(LoBit) << ")" << ", " + << Offset << ");\n"; HighScratchAccess = std::max(HighScratchAccess, NumBits + LoBit); } diff --git a/llvm/utils/TableGen/VarLenCodeEmitterGen.h b/llvm/utils/TableGen/Common/VarLenCodeEmitterGen.h similarity index 100% rename from llvm/utils/TableGen/VarLenCodeEmitterGen.h rename to llvm/utils/TableGen/Common/VarLenCodeEmitterGen.h diff --git a/llvm/utils/TableGen/CompressInstEmitter.cpp b/llvm/utils/TableGen/CompressInstEmitter.cpp index f703fff0ef3e..fcf77934faac 100644 --- a/llvm/utils/TableGen/CompressInstEmitter.cpp +++ b/llvm/utils/TableGen/CompressInstEmitter.cpp @@ -64,9 +64,9 @@ //===----------------------------------------------------------------------===// -#include "CodeGenInstruction.h" -#include "CodeGenRegisters.h" -#include "CodeGenTarget.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenRegisters.h" +#include "Common/CodeGenTarget.h" #include "llvm/ADT/IndexedMap.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringMap.h" diff --git a/llvm/utils/TableGen/DAGISelEmitter.cpp b/llvm/utils/TableGen/DAGISelEmitter.cpp index 336cee09b90c..b43a8e659dd9 100644 --- a/llvm/utils/TableGen/DAGISelEmitter.cpp +++ b/llvm/utils/TableGen/DAGISelEmitter.cpp @@ -10,10 +10,10 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenDAGPatterns.h" -#include "CodeGenInstruction.h" -#include "CodeGenTarget.h" -#include "DAGISelMatcher.h" +#include "Common/CodeGenDAGPatterns.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenTarget.h" +#include "Common/DAGISelMatcher.h" #include "llvm/Support/Debug.h" #include "llvm/TableGen/Record.h" #include "llvm/TableGen/TableGenBackend.h" diff --git a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp index 533b8c423690..dcecac4380ce 100644 --- a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp +++ b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp @@ -10,12 +10,12 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenDAGPatterns.h" -#include "CodeGenInstruction.h" -#include "CodeGenRegisters.h" -#include "CodeGenTarget.h" -#include "DAGISelMatcher.h" -#include "SDNodeProperties.h" +#include "Basic/SDNodeProperties.h" +#include "Common/CodeGenDAGPatterns.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenRegisters.h" +#include "Common/CodeGenTarget.h" +#include "Common/DAGISelMatcher.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/MapVector.h" #include "llvm/ADT/StringMap.h" diff --git a/llvm/utils/TableGen/DAGISelMatcherGen.cpp b/llvm/utils/TableGen/DAGISelMatcherGen.cpp index e8bdabaa0c7e..99babdf07316 100644 --- a/llvm/utils/TableGen/DAGISelMatcherGen.cpp +++ b/llvm/utils/TableGen/DAGISelMatcherGen.cpp @@ -6,13 +6,13 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenDAGPatterns.h" -#include "CodeGenInstruction.h" -#include "CodeGenRegisters.h" -#include "CodeGenTarget.h" -#include "DAGISelMatcher.h" -#include "InfoByHwMode.h" -#include "SDNodeProperties.h" +#include "Basic/SDNodeProperties.h" +#include "Common/CodeGenDAGPatterns.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenRegisters.h" +#include "Common/CodeGenTarget.h" +#include "Common/DAGISelMatcher.h" +#include "Common/InfoByHwMode.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringMap.h" #include "llvm/TableGen/Error.h" diff --git a/llvm/utils/TableGen/DAGISelMatcherOpt.cpp b/llvm/utils/TableGen/DAGISelMatcherOpt.cpp index 047d285f9914..224102e49d98 100644 --- a/llvm/utils/TableGen/DAGISelMatcherOpt.cpp +++ b/llvm/utils/TableGen/DAGISelMatcherOpt.cpp @@ -10,9 +10,9 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenDAGPatterns.h" -#include "DAGISelMatcher.h" -#include "SDNodeProperties.h" +#include "Basic/SDNodeProperties.h" +#include "Common/CodeGenDAGPatterns.h" +#include "Common/DAGISelMatcher.h" #include "llvm/ADT/StringSet.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" diff --git a/llvm/utils/TableGen/DFAEmitter.cpp b/llvm/utils/TableGen/DFAEmitter.cpp index ce8cc2a078d7..567184d3d5ee 100644 --- a/llvm/utils/TableGen/DFAEmitter.cpp +++ b/llvm/utils/TableGen/DFAEmitter.cpp @@ -21,7 +21,7 @@ //===----------------------------------------------------------------------===// #include "DFAEmitter.h" -#include "SequenceToOffsetTable.h" +#include "Basic/SequenceToOffsetTable.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringExtras.h" #include "llvm/ADT/UniqueVector.h" diff --git a/llvm/utils/TableGen/DFAPacketizerEmitter.cpp b/llvm/utils/TableGen/DFAPacketizerEmitter.cpp index 26ea1846ffae..3c74df048660 100644 --- a/llvm/utils/TableGen/DFAPacketizerEmitter.cpp +++ b/llvm/utils/TableGen/DFAPacketizerEmitter.cpp @@ -14,8 +14,8 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenSchedule.h" -#include "CodeGenTarget.h" +#include "Common/CodeGenSchedule.h" +#include "Common/CodeGenTarget.h" #include "DFAEmitter.h" #include "llvm/ADT/SmallVector.h" #include "llvm/Support/Debug.h" diff --git a/llvm/utils/TableGen/DXILEmitter.cpp b/llvm/utils/TableGen/DXILEmitter.cpp index af1efb8aa99f..47c89df35e19 100644 --- a/llvm/utils/TableGen/DXILEmitter.cpp +++ b/llvm/utils/TableGen/DXILEmitter.cpp @@ -11,8 +11,8 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenTarget.h" -#include "SequenceToOffsetTable.h" +#include "Basic/SequenceToOffsetTable.h" +#include "Common/CodeGenTarget.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/SmallVector.h" diff --git a/llvm/utils/TableGen/DecoderEmitter.cpp b/llvm/utils/TableGen/DecoderEmitter.cpp index 732f34ed04c5..494dc93faace 100644 --- a/llvm/utils/TableGen/DecoderEmitter.cpp +++ b/llvm/utils/TableGen/DecoderEmitter.cpp @@ -11,12 +11,12 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenHwModes.h" -#include "CodeGenInstruction.h" -#include "CodeGenTarget.h" -#include "InfoByHwMode.h" +#include "Common/CodeGenHwModes.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenTarget.h" +#include "Common/InfoByHwMode.h" +#include "Common/VarLenCodeEmitterGen.h" #include "TableGenBackends.h" -#include "VarLenCodeEmitterGen.h" #include "llvm/ADT/APInt.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/CachedHashString.h" diff --git a/llvm/utils/TableGen/DisassemblerEmitter.cpp b/llvm/utils/TableGen/DisassemblerEmitter.cpp index 2d653af4d302..d41750075b41 100644 --- a/llvm/utils/TableGen/DisassemblerEmitter.cpp +++ b/llvm/utils/TableGen/DisassemblerEmitter.cpp @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenTarget.h" +#include "Common/CodeGenTarget.h" #include "TableGenBackends.h" #include "WebAssemblyDisassemblerEmitter.h" #include "X86DisassemblerTables.h" diff --git a/llvm/utils/TableGen/FastISelEmitter.cpp b/llvm/utils/TableGen/FastISelEmitter.cpp index f04c6e3b3bf0..acfdc20316b7 100644 --- a/llvm/utils/TableGen/FastISelEmitter.cpp +++ b/llvm/utils/TableGen/FastISelEmitter.cpp @@ -16,11 +16,11 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenDAGPatterns.h" -#include "CodeGenInstruction.h" -#include "CodeGenRegisters.h" -#include "CodeGenTarget.h" -#include "InfoByHwMode.h" +#include "Common/CodeGenDAGPatterns.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenRegisters.h" +#include "Common/CodeGenTarget.h" +#include "Common/InfoByHwMode.h" #include "llvm/ADT/StringSwitch.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/TableGen/Error.h" diff --git a/llvm/utils/TableGen/GlobalISel/CMakeLists.txt b/llvm/utils/TableGen/GlobalISel/CMakeLists.txt deleted file mode 100644 index 7262c4058399..000000000000 --- a/llvm/utils/TableGen/GlobalISel/CMakeLists.txt +++ /dev/null @@ -1,20 +0,0 @@ -set(LLVM_LINK_COMPONENTS - Support - TableGen - ) - -add_llvm_library(LLVMTableGenGlobalISel STATIC DISABLE_LLVM_LINK_LLVM_DYLIB - CodeExpander.cpp - CXXPredicates.cpp - MatchDataInfo.cpp - Patterns.cpp - - DEPENDS - vt_gen - ) - -# Users may include its headers as "GlobalISel/*.h" -target_include_directories(LLVMTableGenGlobalISel - INTERFACE - $ - ) diff --git a/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp b/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp index dee3cb4d71a4..39b9f8a2ae17 100644 --- a/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp +++ b/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp @@ -26,18 +26,18 @@ /// //===----------------------------------------------------------------------===// -#include "CodeGenInstruction.h" -#include "CodeGenIntrinsics.h" -#include "CodeGenTarget.h" -#include "GlobalISel/CXXPredicates.h" -#include "GlobalISel/CodeExpander.h" -#include "GlobalISel/CodeExpansions.h" -#include "GlobalISel/CombinerUtils.h" -#include "GlobalISel/MatchDataInfo.h" -#include "GlobalISel/Patterns.h" -#include "GlobalISelMatchTable.h" -#include "GlobalISelMatchTableExecutorEmitter.h" -#include "SubtargetFeatureInfo.h" +#include "Basic/CodeGenIntrinsics.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenTarget.h" +#include "Common/GlobalISel/CXXPredicates.h" +#include "Common/GlobalISel/CodeExpander.h" +#include "Common/GlobalISel/CodeExpansions.h" +#include "Common/GlobalISel/CombinerUtils.h" +#include "Common/GlobalISel/GlobalISelMatchTable.h" +#include "Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.h" +#include "Common/GlobalISel/MatchDataInfo.h" +#include "Common/GlobalISel/Patterns.h" +#include "Common/SubtargetFeatureInfo.h" #include "llvm/ADT/APInt.h" #include "llvm/ADT/EquivalenceClasses.h" #include "llvm/ADT/Hashing.h" diff --git a/llvm/utils/TableGen/GlobalISelEmitter.cpp b/llvm/utils/TableGen/GlobalISelEmitter.cpp index e86057422cd7..25e302ce1ca4 100644 --- a/llvm/utils/TableGen/GlobalISelEmitter.cpp +++ b/llvm/utils/TableGen/GlobalISelEmitter.cpp @@ -30,15 +30,15 @@ /// //===----------------------------------------------------------------------===// -#include "CodeGenDAGPatterns.h" -#include "CodeGenInstruction.h" -#include "CodeGenIntrinsics.h" -#include "CodeGenRegisters.h" -#include "CodeGenTarget.h" -#include "GlobalISelMatchTable.h" -#include "GlobalISelMatchTableExecutorEmitter.h" -#include "InfoByHwMode.h" -#include "SubtargetFeatureInfo.h" +#include "Basic/CodeGenIntrinsics.h" +#include "Common/CodeGenDAGPatterns.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenRegisters.h" +#include "Common/CodeGenTarget.h" +#include "Common/GlobalISel/GlobalISelMatchTable.h" +#include "Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.h" +#include "Common/InfoByHwMode.h" +#include "Common/SubtargetFeatureInfo.h" #include "llvm/ADT/Statistic.h" #include "llvm/CodeGenTypes/LowLevelType.h" #include "llvm/CodeGenTypes/MachineValueType.h" diff --git a/llvm/utils/TableGen/InstrDocsEmitter.cpp b/llvm/utils/TableGen/InstrDocsEmitter.cpp index efabf6bb7ba6..f948540e18db 100644 --- a/llvm/utils/TableGen/InstrDocsEmitter.cpp +++ b/llvm/utils/TableGen/InstrDocsEmitter.cpp @@ -18,9 +18,9 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenDAGPatterns.h" -#include "CodeGenInstruction.h" -#include "CodeGenTarget.h" +#include "Common/CodeGenDAGPatterns.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenTarget.h" #include "llvm/TableGen/Record.h" #include "llvm/TableGen/TableGenBackend.h" #include diff --git a/llvm/utils/TableGen/InstrInfoEmitter.cpp b/llvm/utils/TableGen/InstrInfoEmitter.cpp index 2d08447429d9..36f8fa146539 100644 --- a/llvm/utils/TableGen/InstrInfoEmitter.cpp +++ b/llvm/utils/TableGen/InstrInfoEmitter.cpp @@ -11,15 +11,15 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenDAGPatterns.h" -#include "CodeGenInstruction.h" -#include "CodeGenSchedule.h" -#include "CodeGenTarget.h" -#include "PredicateExpander.h" -#include "SequenceToOffsetTable.h" -#include "SubtargetFeatureInfo.h" +#include "Basic/SequenceToOffsetTable.h" +#include "Common/CodeGenDAGPatterns.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenSchedule.h" +#include "Common/CodeGenTarget.h" +#include "Common/PredicateExpander.h" +#include "Common/SubtargetFeatureInfo.h" +#include "Common/Types.h" #include "TableGenBackends.h" -#include "Types.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" diff --git a/llvm/utils/TableGen/IntrinsicEmitter.cpp b/llvm/utils/TableGen/IntrinsicEmitter.cpp index 50a34eac7ca3..a7e99fa4c050 100644 --- a/llvm/utils/TableGen/IntrinsicEmitter.cpp +++ b/llvm/utils/TableGen/IntrinsicEmitter.cpp @@ -10,8 +10,8 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenIntrinsics.h" -#include "SequenceToOffsetTable.h" +#include "Basic/CodeGenIntrinsics.h" +#include "Basic/SequenceToOffsetTable.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringExtras.h" diff --git a/llvm/utils/TableGen/MacroFusionPredicatorEmitter.cpp b/llvm/utils/TableGen/MacroFusionPredicatorEmitter.cpp index 91c3b0b4359c..e9e63fa8d0de 100644 --- a/llvm/utils/TableGen/MacroFusionPredicatorEmitter.cpp +++ b/llvm/utils/TableGen/MacroFusionPredicatorEmitter.cpp @@ -38,8 +38,8 @@ // //===---------------------------------------------------------------------===// -#include "CodeGenTarget.h" -#include "PredicateExpander.h" +#include "Common/CodeGenTarget.h" +#include "Common/PredicateExpander.h" #include "llvm/Support/Debug.h" #include "llvm/TableGen/Error.h" #include "llvm/TableGen/Record.h" diff --git a/llvm/utils/TableGen/OptParserEmitter.cpp b/llvm/utils/TableGen/OptParserEmitter.cpp index c25f6c59cab3..6334af53f88f 100644 --- a/llvm/utils/TableGen/OptParserEmitter.cpp +++ b/llvm/utils/TableGen/OptParserEmitter.cpp @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// -#include "OptEmitter.h" +#include "Common/OptEmitter.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/Twine.h" diff --git a/llvm/utils/TableGen/OptRSTEmitter.cpp b/llvm/utils/TableGen/OptRSTEmitter.cpp index 5a7f079dc168..75b7cbdf2988 100644 --- a/llvm/utils/TableGen/OptRSTEmitter.cpp +++ b/llvm/utils/TableGen/OptRSTEmitter.cpp @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// -#include "OptEmitter.h" +#include "Common/OptEmitter.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringMap.h" #include "llvm/TableGen/Record.h" diff --git a/llvm/utils/TableGen/PseudoLoweringEmitter.cpp b/llvm/utils/TableGen/PseudoLoweringEmitter.cpp index 7f692f29192d..01cfd4a1d982 100644 --- a/llvm/utils/TableGen/PseudoLoweringEmitter.cpp +++ b/llvm/utils/TableGen/PseudoLoweringEmitter.cpp @@ -6,8 +6,8 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenInstruction.h" -#include "CodeGenTarget.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenTarget.h" #include "llvm/ADT/IndexedMap.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringMap.h" diff --git a/llvm/utils/TableGen/RegisterBankEmitter.cpp b/llvm/utils/TableGen/RegisterBankEmitter.cpp index 8b59411c5bc3..5546e727af38 100644 --- a/llvm/utils/TableGen/RegisterBankEmitter.cpp +++ b/llvm/utils/TableGen/RegisterBankEmitter.cpp @@ -11,9 +11,9 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenRegisters.h" -#include "CodeGenTarget.h" -#include "InfoByHwMode.h" +#include "Common/CodeGenRegisters.h" +#include "Common/CodeGenTarget.h" +#include "Common/InfoByHwMode.h" #include "llvm/ADT/BitVector.h" #include "llvm/Support/Debug.h" #include "llvm/TableGen/Error.h" diff --git a/llvm/utils/TableGen/RegisterInfoEmitter.cpp b/llvm/utils/TableGen/RegisterInfoEmitter.cpp index c4fc1930488c..a1259bff6ba8 100644 --- a/llvm/utils/TableGen/RegisterInfoEmitter.cpp +++ b/llvm/utils/TableGen/RegisterInfoEmitter.cpp @@ -12,12 +12,12 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenHwModes.h" -#include "CodeGenRegisters.h" -#include "CodeGenTarget.h" -#include "InfoByHwMode.h" -#include "SequenceToOffsetTable.h" -#include "Types.h" +#include "Basic/SequenceToOffsetTable.h" +#include "Common/CodeGenHwModes.h" +#include "Common/CodeGenRegisters.h" +#include "Common/CodeGenTarget.h" +#include "Common/InfoByHwMode.h" +#include "Common/Types.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/BitVector.h" #include "llvm/ADT/STLExtras.h" diff --git a/llvm/utils/TableGen/SearchableTableEmitter.cpp b/llvm/utils/TableGen/SearchableTableEmitter.cpp index 51f18f360ed3..48ee23db957d 100644 --- a/llvm/utils/TableGen/SearchableTableEmitter.cpp +++ b/llvm/utils/TableGen/SearchableTableEmitter.cpp @@ -13,8 +13,8 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenIntrinsics.h" -#include "CodeGenTarget.h" +#include "Basic/CodeGenIntrinsics.h" +#include "Common/CodeGenTarget.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/STLExtras.h" diff --git a/llvm/utils/TableGen/SubtargetEmitter.cpp b/llvm/utils/TableGen/SubtargetEmitter.cpp index d350d7de139f..2e2c57b802ee 100644 --- a/llvm/utils/TableGen/SubtargetEmitter.cpp +++ b/llvm/utils/TableGen/SubtargetEmitter.cpp @@ -10,10 +10,10 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenHwModes.h" -#include "CodeGenSchedule.h" -#include "CodeGenTarget.h" -#include "PredicateExpander.h" +#include "Common/CodeGenHwModes.h" +#include "Common/CodeGenSchedule.h" +#include "Common/CodeGenTarget.h" +#include "Common/PredicateExpander.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/StringExtras.h" diff --git a/llvm/utils/TableGen/WebAssemblyDisassemblerEmitter.cpp b/llvm/utils/TableGen/WebAssemblyDisassemblerEmitter.cpp index 928129f24fcb..e9436ab16e44 100644 --- a/llvm/utils/TableGen/WebAssemblyDisassemblerEmitter.cpp +++ b/llvm/utils/TableGen/WebAssemblyDisassemblerEmitter.cpp @@ -14,7 +14,7 @@ //===----------------------------------------------------------------------===// #include "WebAssemblyDisassemblerEmitter.h" -#include "CodeGenInstruction.h" +#include "Common/CodeGenInstruction.h" #include "llvm/ADT/STLExtras.h" #include "llvm/Support/raw_ostream.h" #include "llvm/TableGen/Record.h" diff --git a/llvm/utils/TableGen/X86CompressEVEXTablesEmitter.cpp b/llvm/utils/TableGen/X86CompressEVEXTablesEmitter.cpp index 0a9abbfe186e..c721502a395f 100644 --- a/llvm/utils/TableGen/X86CompressEVEXTablesEmitter.cpp +++ b/llvm/utils/TableGen/X86CompressEVEXTablesEmitter.cpp @@ -11,8 +11,8 @@ /// //===----------------------------------------------------------------------===// -#include "CodeGenInstruction.h" -#include "CodeGenTarget.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenTarget.h" #include "X86RecognizableInstr.h" #include "llvm/TableGen/Error.h" #include "llvm/TableGen/Record.h" diff --git a/llvm/utils/TableGen/X86FoldTablesEmitter.cpp b/llvm/utils/TableGen/X86FoldTablesEmitter.cpp index 1319042e48d0..5871e678b16e 100644 --- a/llvm/utils/TableGen/X86FoldTablesEmitter.cpp +++ b/llvm/utils/TableGen/X86FoldTablesEmitter.cpp @@ -11,8 +11,8 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenInstruction.h" -#include "CodeGenTarget.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenTarget.h" #include "X86RecognizableInstr.h" #include "llvm/ADT/StringSwitch.h" #include "llvm/Support/FormattedStream.h" diff --git a/llvm/utils/TableGen/X86MnemonicTables.cpp b/llvm/utils/TableGen/X86MnemonicTables.cpp index aeafee157462..d9ceed40f7c7 100644 --- a/llvm/utils/TableGen/X86MnemonicTables.cpp +++ b/llvm/utils/TableGen/X86MnemonicTables.cpp @@ -11,8 +11,8 @@ // //===----------------------------------------------------------------------===// -#include "CodeGenInstruction.h" -#include "CodeGenTarget.h" +#include "Common/CodeGenInstruction.h" +#include "Common/CodeGenTarget.h" #include "X86RecognizableInstr.h" #include "llvm/TableGen/Record.h" #include "llvm/TableGen/TableGenBackend.h" diff --git a/llvm/utils/TableGen/X86RecognizableInstr.h b/llvm/utils/TableGen/X86RecognizableInstr.h index 68af68fb5aa0..12fb41750cb3 100644 --- a/llvm/utils/TableGen/X86RecognizableInstr.h +++ b/llvm/utils/TableGen/X86RecognizableInstr.h @@ -16,7 +16,7 @@ #ifndef LLVM_UTILS_TABLEGEN_X86RECOGNIZABLEINSTR_H #define LLVM_UTILS_TABLEGEN_X86RECOGNIZABLEINSTR_H -#include "CodeGenInstruction.h" +#include "Common/CodeGenInstruction.h" #include "llvm/Support/X86DisassemblerDecoderCommon.h" #include #include -- GitLab From aa962d67ee896f416e285a9298e45fc08ff95eef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= Date: Mon, 25 Mar 2024 09:42:41 +0100 Subject: [PATCH 187/306] [clang-repl] Fix Value for platforms where unqualified char is unsigned (#86118) Signedness of unqualified `char` is unspecified and varies between platforms. This patch adds `Char_U` in `REPL_BUILTIN_TYPES` to account for platforms that default to `unsigned char`. --- clang/include/clang/Interpreter/Value.h | 1 + clang/unittests/Interpreter/InterpreterTest.cpp | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/clang/include/clang/Interpreter/Value.h b/clang/include/clang/Interpreter/Value.h index c380cd91550d..d70e8f871902 100644 --- a/clang/include/clang/Interpreter/Value.h +++ b/clang/include/clang/Interpreter/Value.h @@ -76,6 +76,7 @@ class QualType; X(bool, Bool) \ X(char, Char_S) \ X(signed char, SChar) \ + X(unsigned char, Char_U) \ X(unsigned char, UChar) \ X(short, Short) \ X(unsigned short, UShort) \ diff --git a/clang/unittests/Interpreter/InterpreterTest.cpp b/clang/unittests/Interpreter/InterpreterTest.cpp index e76c0677db5e..69bc2da24288 100644 --- a/clang/unittests/Interpreter/InterpreterTest.cpp +++ b/clang/unittests/Interpreter/InterpreterTest.cpp @@ -340,6 +340,12 @@ TEST(InterpreterTest, Value) { EXPECT_EQ(V1.getKind(), Value::K_Int); EXPECT_FALSE(V1.isManuallyAlloc()); + Value V1b; + llvm::cantFail(Interp->ParseAndExecute("char c = 42;")); + llvm::cantFail(Interp->ParseAndExecute("c", &V1b)); + EXPECT_TRUE(V1b.getKind() == Value::K_Char_S || + V1b.getKind() == Value::K_Char_U); + Value V2; llvm::cantFail(Interp->ParseAndExecute("double y = 3.14;")); llvm::cantFail(Interp->ParseAndExecute("y", &V2)); -- GitLab From 0cf4788d9d0df60980cb48d28aafe7a86aa15a14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= Date: Mon, 25 Mar 2024 09:44:25 +0100 Subject: [PATCH 188/306] [clang-repl] Factor out CreateJITBuilder() and allow specialization in derived classes (#84461) The LLJITBuilder interface provides a very convenient way to configure the ORCv2 JIT engine. IncrementalExecutor already used it internally to construct the JIT, but didn't provide external access. This patch lifts control of the creation process to the Interpreter and allows injection of a custom instance through the extended interface. The Interpreter's default behavior remains unchanged and the IncrementalExecutor remains an implementation detail. --- clang/include/clang/Interpreter/Interpreter.h | 8 ++ clang/lib/Interpreter/IncrementalExecutor.cpp | 33 ++--- clang/lib/Interpreter/IncrementalExecutor.h | 9 +- clang/lib/Interpreter/Interpreter.cpp | 26 +++- .../Interpreter/InterpreterExtensionsTest.cpp | 121 +++++++++++++++++- 5 files changed, 175 insertions(+), 22 deletions(-) diff --git a/clang/include/clang/Interpreter/Interpreter.h b/clang/include/clang/Interpreter/Interpreter.h index 1dcba1ef9679..970e0245417b 100644 --- a/clang/include/clang/Interpreter/Interpreter.h +++ b/clang/include/clang/Interpreter/Interpreter.h @@ -30,6 +30,7 @@ namespace llvm { namespace orc { class LLJIT; +class LLJITBuilder; class ThreadSafeContext; } // namespace orc } // namespace llvm @@ -127,6 +128,13 @@ protected: // custom runtime. virtual std::unique_ptr FindRuntimeInterface(); + // Lazily construct thev ORCv2 JITBuilder. This called when the internal + // IncrementalExecutor is created. The default implementation populates an + // in-process JIT with debugging support. Override this to configure the JIT + // engine used for execution. + virtual llvm::Expected> + CreateJITBuilder(CompilerInstance &CI); + public: virtual ~Interpreter(); diff --git a/clang/lib/Interpreter/IncrementalExecutor.cpp b/clang/lib/Interpreter/IncrementalExecutor.cpp index 40bcef94797d..6f036107c14a 100644 --- a/clang/lib/Interpreter/IncrementalExecutor.cpp +++ b/clang/lib/Interpreter/IncrementalExecutor.cpp @@ -20,6 +20,7 @@ #include "llvm/ExecutionEngine/Orc/Debugging/DebuggerSupport.h" #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" #include "llvm/ExecutionEngine/Orc/IRCompileLayer.h" +#include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h" #include "llvm/ExecutionEngine/Orc/LLJIT.h" #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" #include "llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.h" @@ -36,26 +37,28 @@ LLVM_ATTRIBUTE_USED void linkComponents() { namespace clang { +llvm::Expected> +IncrementalExecutor::createDefaultJITBuilder( + llvm::orc::JITTargetMachineBuilder JTMB) { + auto JITBuilder = std::make_unique(); + JITBuilder->setJITTargetMachineBuilder(std::move(JTMB)); + JITBuilder->setPrePlatformSetup([](llvm::orc::LLJIT &J) { + // Try to enable debugging of JIT'd code (only works with JITLink for + // ELF and MachO). + consumeError(llvm::orc::enableDebuggerSupport(J)); + return llvm::Error::success(); + }); + return std::move(JITBuilder); +} + IncrementalExecutor::IncrementalExecutor(llvm::orc::ThreadSafeContext &TSC, - llvm::Error &Err, - const clang::TargetInfo &TI) + llvm::orc::LLJITBuilder &JITBuilder, + llvm::Error &Err) : TSCtx(TSC) { using namespace llvm::orc; llvm::ErrorAsOutParameter EAO(&Err); - auto JTMB = JITTargetMachineBuilder(TI.getTriple()); - JTMB.addFeatures(TI.getTargetOpts().Features); - LLJITBuilder Builder; - Builder.setJITTargetMachineBuilder(JTMB); - Builder.setPrePlatformSetup( - [](LLJIT &J) { - // Try to enable debugging of JIT'd code (only works with JITLink for - // ELF and MachO). - consumeError(enableDebuggerSupport(J)); - return llvm::Error::success(); - }); - - if (auto JitOrErr = Builder.create()) + if (auto JitOrErr = JITBuilder.create()) Jit = std::move(*JitOrErr); else { Err = JitOrErr.takeError(); diff --git a/clang/lib/Interpreter/IncrementalExecutor.h b/clang/lib/Interpreter/IncrementalExecutor.h index dd0a210a0614..b4347209e14f 100644 --- a/clang/lib/Interpreter/IncrementalExecutor.h +++ b/clang/lib/Interpreter/IncrementalExecutor.h @@ -23,7 +23,9 @@ namespace llvm { class Error; namespace orc { +class JITTargetMachineBuilder; class LLJIT; +class LLJITBuilder; class ThreadSafeContext; } // namespace orc } // namespace llvm @@ -44,8 +46,8 @@ class IncrementalExecutor { public: enum SymbolNameKind { IRName, LinkerName }; - IncrementalExecutor(llvm::orc::ThreadSafeContext &TSC, llvm::Error &Err, - const clang::TargetInfo &TI); + IncrementalExecutor(llvm::orc::ThreadSafeContext &TSC, + llvm::orc::LLJITBuilder &JITBuilder, llvm::Error &Err); ~IncrementalExecutor(); llvm::Error addModule(PartialTranslationUnit &PTU); @@ -56,6 +58,9 @@ public: getSymbolAddress(llvm::StringRef Name, SymbolNameKind NameKind) const; llvm::orc::LLJIT &GetExecutionEngine() { return *Jit; } + + static llvm::Expected> + createDefaultJITBuilder(llvm::orc::JITTargetMachineBuilder JTMB); }; } // end namespace clang diff --git a/clang/lib/Interpreter/Interpreter.cpp b/clang/lib/Interpreter/Interpreter.cpp index 7fa52f2f15fc..cf31456b6950 100644 --- a/clang/lib/Interpreter/Interpreter.cpp +++ b/clang/lib/Interpreter/Interpreter.cpp @@ -372,15 +372,35 @@ Interpreter::Parse(llvm::StringRef Code) { return IncrParser->Parse(Code); } +static llvm::Expected +createJITTargetMachineBuilder(const std::string &TT) { + if (TT == llvm::sys::getProcessTriple()) + // This fails immediately if the target backend is not registered + return llvm::orc::JITTargetMachineBuilder::detectHost(); + + // If the target backend is not registered, LLJITBuilder::create() will fail + return llvm::orc::JITTargetMachineBuilder(llvm::Triple(TT)); +} + +llvm::Expected> +Interpreter::CreateJITBuilder(CompilerInstance &CI) { + auto JTMB = createJITTargetMachineBuilder(CI.getTargetOpts().Triple); + if (!JTMB) + return JTMB.takeError(); + return IncrementalExecutor::createDefaultJITBuilder(std::move(*JTMB)); +} + llvm::Error Interpreter::CreateExecutor() { - const clang::TargetInfo &TI = - getCompilerInstance()->getASTContext().getTargetInfo(); if (IncrExecutor) return llvm::make_error("Operation failed. " "Execution engine exists", std::error_code()); + llvm::Expected> JB = + CreateJITBuilder(*getCompilerInstance()); + if (!JB) + return JB.takeError(); llvm::Error Err = llvm::Error::success(); - auto Executor = std::make_unique(*TSCtx, Err, TI); + auto Executor = std::make_unique(*TSCtx, **JB, Err); if (!Err) IncrExecutor = std::move(Executor); diff --git a/clang/unittests/Interpreter/InterpreterExtensionsTest.cpp b/clang/unittests/Interpreter/InterpreterExtensionsTest.cpp index b7708616fd24..8bc429d9ec2d 100644 --- a/clang/unittests/Interpreter/InterpreterExtensionsTest.cpp +++ b/clang/unittests/Interpreter/InterpreterExtensionsTest.cpp @@ -18,14 +18,21 @@ #include "clang/Sema/Sema.h" #include "llvm/ExecutionEngine/Orc/LLJIT.h" +#include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h" #include "llvm/Support/Error.h" #include "llvm/Support/TargetSelect.h" +#include "llvm/Support/Threading.h" #include "llvm/Testing/Support/Error.h" #include "gmock/gmock.h" #include "gtest/gtest.h" + #include +#if defined(_AIX) +#define CLANG_INTERPRETER_PLATFORM_CANNOT_CREATE_LLJIT +#endif + using namespace clang; namespace { @@ -41,6 +48,10 @@ struct LLVMInitRAII { LLVMInitRAII() { llvm::InitializeNativeTarget(); llvm::InitializeNativeTargetAsmPrinter(); + LLVMInitializeARMTarget(); + LLVMInitializeARMTargetInfo(); + LLVMInitializeARMTargetMC(); + LLVMInitializeARMAsmPrinter(); } ~LLVMInitRAII() { llvm::llvm_shutdown(); } } LLVMInit; @@ -51,12 +62,30 @@ public: llvm::Error &Err) : Interpreter(std::move(CI), Err) {} - llvm::Error testCreateExecutor() { return Interpreter::CreateExecutor(); } + llvm::Error testCreateJITBuilderError() { + JB = nullptr; + return Interpreter::CreateExecutor(); + } + + llvm::Error testCreateExecutor() { + JB = std::make_unique(); + return Interpreter::CreateExecutor(); + } void resetExecutor() { Interpreter::ResetExecutor(); } + +private: + llvm::Expected> + CreateJITBuilder(CompilerInstance &CI) override { + if (JB) + return std::move(JB); + return llvm::make_error("TestError", std::error_code()); + } + + std::unique_ptr JB; }; -#ifdef _AIX +#ifdef CLANG_INTERPRETER_PLATFORM_CANNOT_CREATE_LLJIT TEST(InterpreterExtensionsTest, DISABLED_ExecutorCreateReset) { #else TEST(InterpreterExtensionsTest, ExecutorCreateReset) { @@ -69,6 +98,8 @@ TEST(InterpreterExtensionsTest, ExecutorCreateReset) { llvm::Error ErrOut = llvm::Error::success(); TestCreateResetExecutor Interp(cantFail(CB.CreateCpp()), ErrOut); cantFail(std::move(ErrOut)); + EXPECT_THAT_ERROR(Interp.testCreateJITBuilderError(), + llvm::FailedWithMessage("TestError")); cantFail(Interp.testCreateExecutor()); Interp.resetExecutor(); cantFail(Interp.testCreateExecutor()); @@ -126,4 +157,90 @@ TEST(InterpreterExtensionsTest, FindRuntimeInterface) { EXPECT_EQ(1U, Interp.RuntimeIBPtr->TransformerQueries); } +class CustomJBInterpreter : public Interpreter { + using CustomJITBuilderCreatorFunction = + std::function>()>; + CustomJITBuilderCreatorFunction JBCreator = nullptr; + +public: + CustomJBInterpreter(std::unique_ptr CI, llvm::Error &ErrOut) + : Interpreter(std::move(CI), ErrOut) {} + + ~CustomJBInterpreter() override { + // Skip cleanUp() because it would trigger LLJIT default dtors + Interpreter::ResetExecutor(); + } + + void setCustomJITBuilderCreator(CustomJITBuilderCreatorFunction Fn) { + JBCreator = std::move(Fn); + } + + llvm::Expected> + CreateJITBuilder(CompilerInstance &CI) override { + if (JBCreator) + return JBCreator(); + return Interpreter::CreateJITBuilder(CI); + } + + llvm::Error CreateExecutor() { return Interpreter::CreateExecutor(); } +}; + +#ifdef CLANG_INTERPRETER_PLATFORM_CANNOT_CREATE_LLJIT +TEST(InterpreterExtensionsTest, DISABLED_DefaultCrossJIT) { +#else +TEST(InterpreterExtensionsTest, DefaultCrossJIT) { +#endif + IncrementalCompilerBuilder CB; + CB.SetTargetTriple("armv6-none-eabi"); + auto CI = cantFail(CB.CreateCpp()); + llvm::Error ErrOut = llvm::Error::success(); + CustomJBInterpreter Interp(std::move(CI), ErrOut); + cantFail(std::move(ErrOut)); + cantFail(Interp.CreateExecutor()); +} + +#ifdef CLANG_INTERPRETER_PLATFORM_CANNOT_CREATE_LLJIT +TEST(InterpreterExtensionsTest, DISABLED_CustomCrossJIT) { +#else +TEST(InterpreterExtensionsTest, CustomCrossJIT) { +#endif + std::string TargetTriple = "armv6-none-eabi"; + + IncrementalCompilerBuilder CB; + CB.SetTargetTriple(TargetTriple); + auto CI = cantFail(CB.CreateCpp()); + llvm::Error ErrOut = llvm::Error::success(); + CustomJBInterpreter Interp(std::move(CI), ErrOut); + cantFail(std::move(ErrOut)); + + using namespace llvm::orc; + LLJIT *JIT = nullptr; + std::vector> Objs; + Interp.setCustomJITBuilderCreator([&]() { + auto JTMB = JITTargetMachineBuilder(llvm::Triple(TargetTriple)); + JTMB.setCPU("cortex-m0plus"); + auto JB = std::make_unique(); + JB->setJITTargetMachineBuilder(JTMB); + JB->setPlatformSetUp(setUpInactivePlatform); + JB->setNotifyCreatedCallback([&](LLJIT &J) { + ObjectLayer &ObjLayer = J.getObjLinkingLayer(); + auto *JITLinkObjLayer = llvm::dyn_cast(&ObjLayer); + JITLinkObjLayer->setReturnObjectBuffer( + [&Objs](std::unique_ptr MB) { + Objs.push_back(std::move(MB)); + }); + JIT = &J; + return llvm::Error::success(); + }); + return JB; + }); + + EXPECT_EQ(0U, Objs.size()); + cantFail(Interp.CreateExecutor()); + cantFail(Interp.ParseAndExecute("int a = 1;")); + ExecutorAddr Addr = cantFail(JIT->lookup("a")); + EXPECT_NE(0U, Addr.getValue()); + EXPECT_EQ(1U, Objs.size()); +} + } // end anonymous namespace -- GitLab From 75e528fdd9594ecb6fdb5d9e7bee1506f7e43be0 Mon Sep 17 00:00:00 2001 From: David Stuttard Date: Mon, 25 Mar 2024 09:01:46 +0000 Subject: [PATCH 189/306] [AMDGPU] Extend zero initialization of return values for TFE (#85759) buffer_load instructions that use TFE also need to zero initialize return values similar to how the image instructions currently work. Add support for this with standard zero init of all results + zero init of just TFE flag when enable-prt-strict-null subtarget feature is disabled. --- .../AMDGPU/AMDGPUInstructionSelector.cpp | 32 -- llvm/lib/Target/AMDGPU/BUFInstructions.td | 5 +- llvm/lib/Target/AMDGPU/MIMGInstructions.td | 1 + llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 87 ++--- llvm/lib/Target/AMDGPU/SIISelLowering.h | 2 +- .../Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp | 6 + llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h | 3 + .../llvm.amdgcn.struct.buffer.load.format.ll | 1 + ...vm.amdgcn.struct.ptr.buffer.load.format.ll | 1 + .../AMDGPU/llvm.amdgcn.image.msaa.load.ll | 111 ++++-- .../llvm.amdgcn.struct.buffer.load.format.ll | 317 +++++++++++++++++- ...vm.amdgcn.struct.ptr.buffer.load.format.ll | 280 ++++++++++++++++ 12 files changed, 741 insertions(+), 105 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp index 94cc1d90e0ca..e13c13913d4e 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp @@ -2045,38 +2045,6 @@ bool AMDGPUInstructionSelector::selectImageIntrinsic( if (BaseOpcode->HasD16) MIB.addImm(IsD16 ? -1 : 0); - if (IsTexFail) { - // An image load instruction with TFE/LWE only conditionally writes to its - // result registers. Initialize them to zero so that we always get well - // defined result values. - assert(VDataOut && !VDataIn); - Register Tied = MRI->cloneVirtualRegister(VDataOut); - Register Zero = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); - BuildMI(*MBB, *MIB, DL, TII.get(AMDGPU::V_MOV_B32_e32), Zero) - .addImm(0); - auto Parts = TRI.getRegSplitParts(MRI->getRegClass(Tied), 4); - if (STI.usePRTStrictNull()) { - // With enable-prt-strict-null enabled, initialize all result registers to - // zero. - auto RegSeq = - BuildMI(*MBB, *MIB, DL, TII.get(AMDGPU::REG_SEQUENCE), Tied); - for (auto Sub : Parts) - RegSeq.addReg(Zero).addImm(Sub); - } else { - // With enable-prt-strict-null disabled, only initialize the extra TFE/LWE - // result register. - Register Undef = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); - BuildMI(*MBB, *MIB, DL, TII.get(AMDGPU::IMPLICIT_DEF), Undef); - auto RegSeq = - BuildMI(*MBB, *MIB, DL, TII.get(AMDGPU::REG_SEQUENCE), Tied); - for (auto Sub : Parts.drop_back(1)) - RegSeq.addReg(Undef).addImm(Sub); - RegSeq.addReg(Zero).addImm(Parts.back()); - } - MIB.addReg(Tied, RegState::Implicit); - MIB->tieOperands(0, MIB->getNumOperands() - 1); - } - MI.eraseFromParent(); constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI); TII.enforceOperandRCAlignment(*MIB, AMDGPU::OpName::vaddr); diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td index 4ae514ffcf78..273f92abf354 100644 --- a/llvm/lib/Target/AMDGPU/BUFInstructions.td +++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td @@ -86,7 +86,7 @@ class BUF_Pseudo has_soffset = 1; bits<1> has_offset = 1; bits<1> has_slc = 1; - bits<1> tfe = ?; + bits<1> tfe = 0; bits<4> elements = 0; bits<1> has_sccb = 1; bits<1> sccb_value = 0; @@ -323,6 +323,7 @@ class MUBUF_Pseudo (MUBUFGetBaseOpcode.ret); let MUBUF = 1; let AsmMatchConverter = "cvtMubuf"; + let usesCustomInserter = 1; } class MUBUF_Real : @@ -3369,7 +3370,7 @@ def MUBUFInfoTable : GenericTable { let CppTypeName = "MUBUFInfo"; let Fields = [ "Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset", - "IsBufferInv" + "IsBufferInv", "tfe" ]; let PrimaryKey = ["Opcode"]; diff --git a/llvm/lib/Target/AMDGPU/MIMGInstructions.td b/llvm/lib/Target/AMDGPU/MIMGInstructions.td index 595ef39ce03e..23e8be0d5e45 100644 --- a/llvm/lib/Target/AMDGPU/MIMGInstructions.td +++ b/llvm/lib/Target/AMDGPU/MIMGInstructions.td @@ -210,6 +210,7 @@ class MIMG : MIMG_Base { let hasPostISelHook = 1; + let usesCustomInserter = 1; Instruction Opcode = !cast(NAME); MIMGBaseOpcode BaseOpcode; diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index d437f339a687..81a231f0cade 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -5410,6 +5410,11 @@ MachineBasicBlock *SITargetLowering::EmitInstrWithCustomInserter( return SplitBB; } default: + if (TII->isImage(MI) || TII->isMUBUF(MI)) { + if (!MI.mayStore()) + AddMemOpInit(MI); + return BB; + } return AMDGPUTargetLowering::EmitInstrWithCustomInserter(MI, BB); } } @@ -15034,60 +15039,67 @@ SDNode *SITargetLowering::PostISelFolding(MachineSDNode *Node, // result register that will be written in the case of a memory access failure. // The required code is also added to tie this init code to the result of the // img instruction. -void SITargetLowering::AddIMGInit(MachineInstr &MI) const { +void SITargetLowering::AddMemOpInit(MachineInstr &MI) const { const SIInstrInfo *TII = getSubtarget()->getInstrInfo(); const SIRegisterInfo &TRI = TII->getRegisterInfo(); MachineRegisterInfo &MRI = MI.getMF()->getRegInfo(); MachineBasicBlock &MBB = *MI.getParent(); - MachineOperand *TFE = TII->getNamedOperand(MI, AMDGPU::OpName::tfe); - MachineOperand *LWE = TII->getNamedOperand(MI, AMDGPU::OpName::lwe); - MachineOperand *D16 = TII->getNamedOperand(MI, AMDGPU::OpName::d16); - - if (!TFE && !LWE) // intersect_ray - return; + int DstIdx = + AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdata); + unsigned InitIdx = 0; - unsigned TFEVal = TFE ? TFE->getImm() : 0; - unsigned LWEVal = LWE ? LWE->getImm() : 0; - unsigned D16Val = D16 ? D16->getImm() : 0; + if (TII->isImage(MI)) { + MachineOperand *TFE = TII->getNamedOperand(MI, AMDGPU::OpName::tfe); + MachineOperand *LWE = TII->getNamedOperand(MI, AMDGPU::OpName::lwe); + MachineOperand *D16 = TII->getNamedOperand(MI, AMDGPU::OpName::d16); - if (!TFEVal && !LWEVal) - return; + if (!TFE && !LWE) // intersect_ray + return; - // At least one of TFE or LWE are non-zero - // We have to insert a suitable initialization of the result value and - // tie this to the dest of the image instruction. + unsigned TFEVal = TFE ? TFE->getImm() : 0; + unsigned LWEVal = LWE ? LWE->getImm() : 0; + unsigned D16Val = D16 ? D16->getImm() : 0; - const DebugLoc &DL = MI.getDebugLoc(); + if (!TFEVal && !LWEVal) + return; - int DstIdx = - AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdata); + // At least one of TFE or LWE are non-zero + // We have to insert a suitable initialization of the result value and + // tie this to the dest of the image instruction. - // Calculate which dword we have to initialize to 0. - MachineOperand *MO_Dmask = TII->getNamedOperand(MI, AMDGPU::OpName::dmask); + // Calculate which dword we have to initialize to 0. + MachineOperand *MO_Dmask = TII->getNamedOperand(MI, AMDGPU::OpName::dmask); - // check that dmask operand is found. - assert(MO_Dmask && "Expected dmask operand in instruction"); + // check that dmask operand is found. + assert(MO_Dmask && "Expected dmask operand in instruction"); - unsigned dmask = MO_Dmask->getImm(); - // Determine the number of active lanes taking into account the - // Gather4 special case - unsigned ActiveLanes = TII->isGather4(MI) ? 4 : llvm::popcount(dmask); + unsigned dmask = MO_Dmask->getImm(); + // Determine the number of active lanes taking into account the + // Gather4 special case + unsigned ActiveLanes = TII->isGather4(MI) ? 4 : llvm::popcount(dmask); - bool Packed = !Subtarget->hasUnpackedD16VMem(); + bool Packed = !Subtarget->hasUnpackedD16VMem(); - unsigned InitIdx = - D16Val && Packed ? ((ActiveLanes + 1) >> 1) + 1 : ActiveLanes + 1; + InitIdx = D16Val && Packed ? ((ActiveLanes + 1) >> 1) + 1 : ActiveLanes + 1; - // Abandon attempt if the dst size isn't large enough - // - this is in fact an error but this is picked up elsewhere and - // reported correctly. - uint32_t DstSize = TRI.getRegSizeInBits(*TII->getOpRegClass(MI, DstIdx)) / 32; - if (DstSize < InitIdx) + // Abandon attempt if the dst size isn't large enough + // - this is in fact an error but this is picked up elsewhere and + // reported correctly. + uint32_t DstSize = + TRI.getRegSizeInBits(*TII->getOpRegClass(MI, DstIdx)) / 32; + if (DstSize < InitIdx) + return; + } else if (TII->isMUBUF(MI) && AMDGPU::getMUBUFTfe(MI.getOpcode())) { + InitIdx = TRI.getRegSizeInBits(*TII->getOpRegClass(MI, DstIdx)) / 32; + } else { return; + } + + const DebugLoc &DL = MI.getDebugLoc(); // Create a register for the initialization value. - Register PrevDst = MRI.createVirtualRegister(TII->getOpRegClass(MI, DstIdx)); + Register PrevDst = MRI.cloneVirtualRegister(MI.getOperand(DstIdx).getReg()); unsigned NewDst = 0; // Final initialized value will be in here // If PRTStrictNull feature is enabled (the default) then initialize @@ -15185,11 +15197,8 @@ void SITargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI, return; } - if (TII->isImage(MI)) { - if (!MI.mayStore()) - AddIMGInit(MI); + if (TII->isImage(MI)) TII->enforceOperandRCAlignment(MI, AMDGPU::OpName::vaddr); - } } static SDValue buildSMovImm32(SelectionDAG &DAG, const SDLoc &DL, diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h index 89da4428e3ab..9856a2923d38 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.h +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h @@ -466,7 +466,7 @@ public: SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override; SDNode *PostISelFolding(MachineSDNode *N, SelectionDAG &DAG) const override; - void AddIMGInit(MachineInstr &MI) const; + void AddMemOpInit(MachineInstr &MI) const; void AdjustInstrPostInstrSelection(MachineInstr &MI, SDNode *Node) const override; diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp index 6d53f68ace70..a90dc32d396f 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp @@ -318,6 +318,7 @@ struct MUBUFInfo { bool has_srsrc; bool has_soffset; bool IsBufferInv; + bool tfe; }; struct MTBUFInfo { @@ -466,6 +467,11 @@ bool getMUBUFIsBufferInv(unsigned Opc) { return Info ? Info->IsBufferInv : false; } +bool getMUBUFTfe(unsigned Opc) { + const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc); + return Info ? Info->tfe : false; +} + bool getSMEMIsBuffer(unsigned Opc) { const SMInfo *Info = getSMEMOpcodeHelper(Opc); return Info ? Info->IsBuffer : false; diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h index 29ac402d9535..f4f9a787100b 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h @@ -525,6 +525,9 @@ bool getMUBUFHasSoffset(unsigned Opc); LLVM_READONLY bool getMUBUFIsBufferInv(unsigned Opc); +LLVM_READONLY +bool getMUBUFTfe(unsigned Opc); + LLVM_READONLY bool getSMEMIsBuffer(unsigned Opc); diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.ll index 686b849ff58f..06bd45a45cce 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.ll @@ -1,6 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py ; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -stop-after=instruction-select -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX8 %s ; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 -stop-after=instruction-select -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX12 %s +; Note that TFE instructions don't have the result initialization to zero due to stopping before finalize-isel - which is where that's inserted define amdgpu_ps float @struct_buffer_load_format_f32__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset(<4 x i32> inreg %rsrc, i32 %vindex, i32 %voffset, i32 inreg %soffset) { ; GFX8-LABEL: name: struct_buffer_load_format_f32__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.load.format.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.load.format.ll index 9edc24554911..1e3f94a5e39c 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.load.format.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.load.format.ll @@ -1,5 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py ; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -stop-after=instruction-select -verify-machineinstrs -o - %s | FileCheck %s +; Note that TFE instructions don't have the result initialization to zero due to stopping before finalize-isel - which is where that's inserted define amdgpu_ps float @struct_ptr_buffer_load_format_f32__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset(ptr addrspace(8) inreg %rsrc, i32 %vindex, i32 %voffset, i32 inreg %soffset) { ; CHECK-LABEL: name: struct_ptr_buffer_load_format_f32__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.msaa.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.msaa.load.ll index 1348315e72e7..7b1f55e7eeba 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.msaa.load.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.msaa.load.ll @@ -22,18 +22,36 @@ main_body: define amdgpu_ps <4 x float> @load_2dmsaa_both(<8 x i32> inreg %rsrc, ptr addrspace(1) inreg %out, i32 %s, i32 %t, i32 %fragid) { ; GFX11-LABEL: load_2dmsaa_both: ; GFX11: ; %bb.0: ; %main_body -; GFX11-NEXT: image_msaa_load v[0:4], v[0:2], s[0:7] dmask:0x2 dim:SQ_RSRC_IMG_2D_MSAA unorm tfe lwe ; encoding: [0x98,0x02,0x60,0xf0,0x00,0x00,0x60,0x00] -; GFX11-NEXT: v_mov_b32_e32 v5, 0 ; encoding: [0x80,0x02,0x0a,0x7e] +; GFX11-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v8, 0 ; encoding: [0x00,0x01,0x10,0xca,0x80,0x00,0x08,0x05] +; GFX11-NEXT: v_dual_mov_b32 v7, v2 :: v_dual_mov_b32 v6, v1 ; encoding: [0x02,0x01,0x10,0xca,0x01,0x01,0x06,0x07] +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4) ; encoding: [0x42,0x02,0x87,0xbf] +; GFX11-NEXT: v_mov_b32_e32 v9, v8 ; encoding: [0x08,0x03,0x12,0x7e] +; GFX11-NEXT: v_mov_b32_e32 v10, v8 ; encoding: [0x08,0x03,0x14,0x7e] +; GFX11-NEXT: v_mov_b32_e32 v11, v8 ; encoding: [0x08,0x03,0x16,0x7e] +; GFX11-NEXT: v_mov_b32_e32 v12, v8 ; encoding: [0x08,0x03,0x18,0x7e] +; GFX11-NEXT: v_dual_mov_b32 v0, v8 :: v_dual_mov_b32 v1, v9 ; encoding: [0x08,0x01,0x10,0xca,0x09,0x01,0x00,0x00] +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) ; encoding: [0x93,0x01,0x87,0xbf] +; GFX11-NEXT: v_dual_mov_b32 v2, v10 :: v_dual_mov_b32 v3, v11 ; encoding: [0x0a,0x01,0x10,0xca,0x0b,0x01,0x02,0x02] +; GFX11-NEXT: v_mov_b32_e32 v4, v12 ; encoding: [0x0c,0x03,0x08,0x7e] +; GFX11-NEXT: image_msaa_load v[0:4], v[5:7], s[0:7] dmask:0x2 dim:SQ_RSRC_IMG_2D_MSAA unorm tfe lwe ; encoding: [0x98,0x02,0x60,0xf0,0x05,0x00,0x60,0x00] ; GFX11-NEXT: s_waitcnt vmcnt(0) ; encoding: [0xf7,0x03,0x89,0xbf] -; GFX11-NEXT: global_store_b32 v5, v4, s[8:9] ; encoding: [0x00,0x00,0x6a,0xdc,0x05,0x04,0x08,0x00] +; GFX11-NEXT: global_store_b32 v8, v4, s[8:9] ; encoding: [0x00,0x00,0x6a,0xdc,0x08,0x04,0x08,0x00] ; GFX11-NEXT: ; return to shader part epilog ; ; GFX12-LABEL: load_2dmsaa_both: ; GFX12: ; %bb.0: ; %main_body -; GFX12-NEXT: image_msaa_load v[0:4], [v0, v1, v2], s[0:7] dmask:0x2 dim:SQ_RSRC_IMG_2D_MSAA unorm tfe lwe ; encoding: [0x0e,0x20,0x86,0xe4,0x00,0x01,0x00,0x00,0x00,0x01,0x02,0x00] -; GFX12-NEXT: v_mov_b32_e32 v5, 0 ; encoding: [0x80,0x02,0x0a,0x7e] +; GFX12-NEXT: v_dual_mov_b32 v7, v0 :: v_dual_mov_b32 v8, 0 ; encoding: [0x00,0x01,0x10,0xca,0x80,0x00,0x08,0x07] +; GFX12-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v6, v1 ; encoding: [0x02,0x01,0x10,0xca,0x01,0x01,0x06,0x05] +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; encoding: [0x22,0x01,0x87,0xbf] +; GFX12-NEXT: v_dual_mov_b32 v9, v8 :: v_dual_mov_b32 v10, v8 ; encoding: [0x08,0x01,0x10,0xca,0x08,0x01,0x0a,0x09] +; GFX12-NEXT: v_dual_mov_b32 v11, v8 :: v_dual_mov_b32 v12, v8 ; encoding: [0x08,0x01,0x10,0xca,0x08,0x01,0x0c,0x0b] +; GFX12-NEXT: v_dual_mov_b32 v0, v8 :: v_dual_mov_b32 v1, v9 ; encoding: [0x08,0x01,0x10,0xca,0x09,0x01,0x00,0x00] +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3) ; encoding: [0x92,0x01,0x87,0xbf] +; GFX12-NEXT: v_dual_mov_b32 v2, v10 :: v_dual_mov_b32 v3, v11 ; encoding: [0x0a,0x01,0x10,0xca,0x0b,0x01,0x02,0x02] +; GFX12-NEXT: v_mov_b32_e32 v4, v12 ; encoding: [0x0c,0x03,0x08,0x7e] +; GFX12-NEXT: image_msaa_load v[0:4], [v7, v6, v5], s[0:7] dmask:0x2 dim:SQ_RSRC_IMG_2D_MSAA unorm tfe lwe ; encoding: [0x0e,0x20,0x86,0xe4,0x00,0x01,0x00,0x00,0x07,0x06,0x05,0x00] ; GFX12-NEXT: s_wait_loadcnt 0x0 ; encoding: [0x00,0x00,0xc0,0xbf] -; GFX12-NEXT: global_store_b32 v5, v4, s[8:9] ; encoding: [0x08,0x80,0x06,0xee,0x00,0x00,0x00,0x02,0x05,0x00,0x00,0x00] +; GFX12-NEXT: global_store_b32 v8, v4, s[8:9] ; encoding: [0x08,0x80,0x06,0xee,0x00,0x00,0x00,0x02,0x08,0x00,0x00,0x00] ; GFX12-NEXT: ; return to shader part epilog main_body: %v = call {<4 x float>,i32} @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32i32.i32(i32 2, i32 %s, i32 %t, i32 %fragid, <8 x i32> %rsrc, i32 3, i32 0) @@ -63,18 +81,37 @@ main_body: define amdgpu_ps <4 x float> @load_2darraymsaa_tfe(<8 x i32> inreg %rsrc, ptr addrspace(1) inreg %out, i32 %s, i32 %t, i32 %slice, i32 %fragid) { ; GFX11-LABEL: load_2darraymsaa_tfe: ; GFX11: ; %bb.0: ; %main_body -; GFX11-NEXT: image_msaa_load v[0:4], v[0:3], s[0:7] dmask:0x8 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm tfe ; encoding: [0x9c,0x08,0x60,0xf0,0x00,0x00,0x20,0x00] -; GFX11-NEXT: v_mov_b32_e32 v5, 0 ; encoding: [0x80,0x02,0x0a,0x7e] +; GFX11-NEXT: v_dual_mov_b32 v9, 0 :: v_dual_mov_b32 v8, v3 ; encoding: [0x80,0x00,0x10,0xca,0x03,0x01,0x08,0x09] +; GFX11-NEXT: v_dual_mov_b32 v7, v2 :: v_dual_mov_b32 v6, v1 ; encoding: [0x02,0x01,0x10,0xca,0x01,0x01,0x06,0x07] +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4) ; encoding: [0x42,0x02,0x87,0xbf] +; GFX11-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v10, v9 ; encoding: [0x00,0x01,0x10,0xca,0x09,0x01,0x0a,0x05] +; GFX11-NEXT: v_mov_b32_e32 v11, v9 ; encoding: [0x09,0x03,0x16,0x7e] +; GFX11-NEXT: v_mov_b32_e32 v12, v9 ; encoding: [0x09,0x03,0x18,0x7e] +; GFX11-NEXT: v_mov_b32_e32 v13, v9 ; encoding: [0x09,0x03,0x1a,0x7e] +; GFX11-NEXT: v_dual_mov_b32 v0, v9 :: v_dual_mov_b32 v1, v10 ; encoding: [0x09,0x01,0x10,0xca,0x0a,0x01,0x00,0x00] +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) ; encoding: [0x93,0x01,0x87,0xbf] +; GFX11-NEXT: v_dual_mov_b32 v2, v11 :: v_dual_mov_b32 v3, v12 ; encoding: [0x0b,0x01,0x10,0xca,0x0c,0x01,0x02,0x02] +; GFX11-NEXT: v_mov_b32_e32 v4, v13 ; encoding: [0x0d,0x03,0x08,0x7e] +; GFX11-NEXT: image_msaa_load v[0:4], v[5:8], s[0:7] dmask:0x8 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm tfe ; encoding: [0x9c,0x08,0x60,0xf0,0x05,0x00,0x20,0x00] ; GFX11-NEXT: s_waitcnt vmcnt(0) ; encoding: [0xf7,0x03,0x89,0xbf] -; GFX11-NEXT: global_store_b32 v5, v4, s[8:9] ; encoding: [0x00,0x00,0x6a,0xdc,0x05,0x04,0x08,0x00] +; GFX11-NEXT: global_store_b32 v9, v4, s[8:9] ; encoding: [0x00,0x00,0x6a,0xdc,0x09,0x04,0x08,0x00] ; GFX11-NEXT: ; return to shader part epilog ; ; GFX12-LABEL: load_2darraymsaa_tfe: ; GFX12: ; %bb.0: ; %main_body -; GFX12-NEXT: image_msaa_load v[0:4], [v0, v1, v2, v3], s[0:7] dmask:0x8 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm tfe ; encoding: [0x0f,0x20,0x06,0xe6,0x00,0x00,0x00,0x00,0x00,0x01,0x02,0x03] -; GFX12-NEXT: v_mov_b32_e32 v5, 0 ; encoding: [0x80,0x02,0x0a,0x7e] +; GFX12-NEXT: v_mov_b32_e32 v9, 0 ; encoding: [0x80,0x02,0x12,0x7e] +; GFX12-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v6, v2 ; encoding: [0x03,0x01,0x10,0xca,0x02,0x01,0x06,0x05] +; GFX12-NEXT: v_dual_mov_b32 v7, v1 :: v_dual_mov_b32 v8, v0 ; encoding: [0x01,0x01,0x10,0xca,0x00,0x01,0x08,0x07] +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; encoding: [0x23,0x01,0x87,0xbf] +; GFX12-NEXT: v_dual_mov_b32 v10, v9 :: v_dual_mov_b32 v11, v9 ; encoding: [0x09,0x01,0x10,0xca,0x09,0x01,0x0a,0x0a] +; GFX12-NEXT: v_dual_mov_b32 v12, v9 :: v_dual_mov_b32 v13, v9 ; encoding: [0x09,0x01,0x10,0xca,0x09,0x01,0x0c,0x0c] +; GFX12-NEXT: v_dual_mov_b32 v0, v9 :: v_dual_mov_b32 v1, v10 ; encoding: [0x09,0x01,0x10,0xca,0x0a,0x01,0x00,0x00] +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3) ; encoding: [0x92,0x01,0x87,0xbf] +; GFX12-NEXT: v_dual_mov_b32 v2, v11 :: v_dual_mov_b32 v3, v12 ; encoding: [0x0b,0x01,0x10,0xca,0x0c,0x01,0x02,0x02] +; GFX12-NEXT: v_mov_b32_e32 v4, v13 ; encoding: [0x0d,0x03,0x08,0x7e] +; GFX12-NEXT: image_msaa_load v[0:4], [v8, v7, v6, v5], s[0:7] dmask:0x8 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm tfe ; encoding: [0x0f,0x20,0x06,0xe6,0x00,0x00,0x00,0x00,0x08,0x07,0x06,0x05] ; GFX12-NEXT: s_wait_loadcnt 0x0 ; encoding: [0x00,0x00,0xc0,0xbf] -; GFX12-NEXT: global_store_b32 v5, v4, s[8:9] ; encoding: [0x08,0x80,0x06,0xee,0x00,0x00,0x00,0x02,0x05,0x00,0x00,0x00] +; GFX12-NEXT: global_store_b32 v9, v4, s[8:9] ; encoding: [0x08,0x80,0x06,0xee,0x00,0x00,0x00,0x02,0x09,0x00,0x00,0x00] ; GFX12-NEXT: ; return to shader part epilog main_body: %v = call {<4 x float>,i32} @llvm.amdgcn.image.msaa.load.2darraymsaa.v4f32i32.i32(i32 8, i32 %s, i32 %t, i32 %slice, i32 %fragid, <8 x i32> %rsrc, i32 1, i32 0) @@ -155,18 +192,31 @@ main_body: define amdgpu_ps <4 x half> @load_2dmsaa_tfe_d16(<8 x i32> inreg %rsrc, ptr addrspace(1) inreg %out, i32 %s, i32 %t, i32 %fragid) { ; GFX11-LABEL: load_2dmsaa_tfe_d16: ; GFX11: ; %bb.0: ; %main_body -; GFX11-NEXT: image_msaa_load v[0:2], v[0:2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm tfe d16 ; encoding: [0x98,0x01,0x62,0xf0,0x00,0x00,0x20,0x00] -; GFX11-NEXT: v_mov_b32_e32 v3, 0 ; encoding: [0x80,0x02,0x06,0x7e] +; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v6, 0 ; encoding: [0x00,0x01,0x10,0xca,0x80,0x00,0x06,0x03] +; GFX11-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1 ; encoding: [0x02,0x01,0x10,0xca,0x01,0x01,0x04,0x05] +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; encoding: [0x22,0x01,0x87,0xbf] +; GFX11-NEXT: v_mov_b32_e32 v7, v6 ; encoding: [0x06,0x03,0x0e,0x7e] +; GFX11-NEXT: v_mov_b32_e32 v8, v6 ; encoding: [0x06,0x03,0x10,0x7e] +; GFX11-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7 ; encoding: [0x06,0x01,0x10,0xca,0x07,0x01,0x00,0x00] +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) ; encoding: [0x02,0x00,0x87,0xbf] +; GFX11-NEXT: v_mov_b32_e32 v2, v8 ; encoding: [0x08,0x03,0x04,0x7e] +; GFX11-NEXT: image_msaa_load v[0:2], v[3:5], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm tfe d16 ; encoding: [0x98,0x01,0x62,0xf0,0x03,0x00,0x20,0x00] ; GFX11-NEXT: s_waitcnt vmcnt(0) ; encoding: [0xf7,0x03,0x89,0xbf] -; GFX11-NEXT: global_store_b32 v3, v2, s[8:9] ; encoding: [0x00,0x00,0x6a,0xdc,0x03,0x02,0x08,0x00] +; GFX11-NEXT: global_store_b32 v6, v2, s[8:9] ; encoding: [0x00,0x00,0x6a,0xdc,0x06,0x02,0x08,0x00] ; GFX11-NEXT: ; return to shader part epilog ; ; GFX12-LABEL: load_2dmsaa_tfe_d16: ; GFX12: ; %bb.0: ; %main_body -; GFX12-NEXT: image_msaa_load v[0:2], [v0, v1, v2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm tfe d16 ; encoding: [0x2e,0x20,0x46,0xe4,0x00,0x00,0x00,0x00,0x00,0x01,0x02,0x00] -; GFX12-NEXT: v_mov_b32_e32 v3, 0 ; encoding: [0x80,0x02,0x06,0x7e] +; GFX12-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v6, 0 ; encoding: [0x00,0x01,0x10,0xca,0x80,0x00,0x06,0x05] +; GFX12-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v4, v1 ; encoding: [0x02,0x01,0x10,0xca,0x01,0x01,0x04,0x03] +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) ; encoding: [0x92,0x00,0x87,0xbf] +; GFX12-NEXT: v_dual_mov_b32 v7, v6 :: v_dual_mov_b32 v8, v6 ; encoding: [0x06,0x01,0x10,0xca,0x06,0x01,0x08,0x07] +; GFX12-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7 ; encoding: [0x06,0x01,0x10,0xca,0x07,0x01,0x00,0x00] +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) ; encoding: [0x02,0x00,0x87,0xbf] +; GFX12-NEXT: v_mov_b32_e32 v2, v8 ; encoding: [0x08,0x03,0x04,0x7e] +; GFX12-NEXT: image_msaa_load v[0:2], [v5, v4, v3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm tfe d16 ; encoding: [0x2e,0x20,0x46,0xe4,0x00,0x00,0x00,0x00,0x05,0x04,0x03,0x00] ; GFX12-NEXT: s_wait_loadcnt 0x0 ; encoding: [0x00,0x00,0xc0,0xbf] -; GFX12-NEXT: global_store_b32 v3, v2, s[8:9] ; encoding: [0x08,0x80,0x06,0xee,0x00,0x00,0x00,0x01,0x03,0x00,0x00,0x00] +; GFX12-NEXT: global_store_b32 v6, v2, s[8:9] ; encoding: [0x08,0x80,0x06,0xee,0x00,0x00,0x00,0x01,0x06,0x00,0x00,0x00] ; GFX12-NEXT: ; return to shader part epilog main_body: %v = call {<4 x half>,i32} @llvm.amdgcn.image.msaa.load.2dmsaa.v4f16i32.i32(i32 1, i32 %s, i32 %t, i32 %fragid, <8 x i32> %rsrc, i32 1, i32 0) @@ -196,18 +246,31 @@ main_body: define amdgpu_ps <4 x half> @load_2darraymsaa_tfe_d16(<8 x i32> inreg %rsrc, ptr addrspace(1) inreg %out, i32 %s, i32 %t, i32 %slice, i32 %fragid) { ; GFX11-LABEL: load_2darraymsaa_tfe_d16: ; GFX11: ; %bb.0: ; %main_body -; GFX11-NEXT: image_msaa_load v[0:2], v[0:3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm tfe d16 ; encoding: [0x9c,0x01,0x62,0xf0,0x00,0x00,0x20,0x00] -; GFX11-NEXT: v_mov_b32_e32 v3, 0 ; encoding: [0x80,0x02,0x06,0x7e] +; GFX11-NEXT: v_dual_mov_b32 v6, v0 :: v_dual_mov_b32 v7, 0 ; encoding: [0x00,0x01,0x10,0xca,0x80,0x00,0x06,0x06] +; GFX11-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v1 ; encoding: [0x02,0x01,0x10,0xca,0x01,0x01,0x04,0x04] +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; encoding: [0x22,0x01,0x87,0xbf] +; GFX11-NEXT: v_mov_b32_e32 v8, v7 ; encoding: [0x07,0x03,0x10,0x7e] +; GFX11-NEXT: v_mov_b32_e32 v9, v7 ; encoding: [0x07,0x03,0x12,0x7e] +; GFX11-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8 ; encoding: [0x07,0x01,0x10,0xca,0x08,0x01,0x00,0x00] +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) ; encoding: [0x02,0x00,0x87,0xbf] +; GFX11-NEXT: v_mov_b32_e32 v2, v9 ; encoding: [0x09,0x03,0x04,0x7e] +; GFX11-NEXT: image_msaa_load v[0:2], [v6, v5, v4, v3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm tfe d16 ; encoding: [0x9d,0x01,0x62,0xf0,0x06,0x00,0x20,0x00,0x05,0x04,0x03,0x00] ; GFX11-NEXT: s_waitcnt vmcnt(0) ; encoding: [0xf7,0x03,0x89,0xbf] -; GFX11-NEXT: global_store_b32 v3, v2, s[8:9] ; encoding: [0x00,0x00,0x6a,0xdc,0x03,0x02,0x08,0x00] +; GFX11-NEXT: global_store_b32 v7, v2, s[8:9] ; encoding: [0x00,0x00,0x6a,0xdc,0x07,0x02,0x08,0x00] ; GFX11-NEXT: ; return to shader part epilog ; ; GFX12-LABEL: load_2darraymsaa_tfe_d16: ; GFX12: ; %bb.0: ; %main_body -; GFX12-NEXT: image_msaa_load v[0:2], [v0, v1, v2, v3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm tfe d16 ; encoding: [0x2f,0x20,0x46,0xe4,0x00,0x00,0x00,0x00,0x00,0x01,0x02,0x03] -; GFX12-NEXT: v_mov_b32_e32 v3, 0 ; encoding: [0x80,0x02,0x06,0x7e] +; GFX12-NEXT: v_dual_mov_b32 v6, v0 :: v_dual_mov_b32 v7, 0 ; encoding: [0x00,0x01,0x10,0xca,0x80,0x00,0x06,0x06] +; GFX12-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v1 ; encoding: [0x02,0x01,0x10,0xca,0x01,0x01,0x04,0x04] +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) ; encoding: [0x92,0x00,0x87,0xbf] +; GFX12-NEXT: v_dual_mov_b32 v8, v7 :: v_dual_mov_b32 v9, v7 ; encoding: [0x07,0x01,0x10,0xca,0x07,0x01,0x08,0x08] +; GFX12-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8 ; encoding: [0x07,0x01,0x10,0xca,0x08,0x01,0x00,0x00] +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) ; encoding: [0x02,0x00,0x87,0xbf] +; GFX12-NEXT: v_mov_b32_e32 v2, v9 ; encoding: [0x09,0x03,0x04,0x7e] +; GFX12-NEXT: image_msaa_load v[0:2], [v6, v5, v4, v3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm tfe d16 ; encoding: [0x2f,0x20,0x46,0xe4,0x00,0x00,0x00,0x00,0x06,0x05,0x04,0x03] ; GFX12-NEXT: s_wait_loadcnt 0x0 ; encoding: [0x00,0x00,0xc0,0xbf] -; GFX12-NEXT: global_store_b32 v3, v2, s[8:9] ; encoding: [0x08,0x80,0x06,0xee,0x00,0x00,0x00,0x01,0x03,0x00,0x00,0x00] +; GFX12-NEXT: global_store_b32 v7, v2, s[8:9] ; encoding: [0x08,0x80,0x06,0xee,0x00,0x00,0x00,0x01,0x07,0x00,0x00,0x00] ; GFX12-NEXT: ; return to shader part epilog main_body: %v = call {<4 x half>,i32} @llvm.amdgcn.image.msaa.load.2darraymsaa.v4f16i32.i32(i32 1, i32 %s, i32 %t, i32 %slice, i32 %fragid, <8 x i32> %rsrc, i32 1, i32 0) diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.ll index 00be32b06de0..ba3d306cc0cf 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.ll @@ -2,6 +2,7 @@ ;RUN: llc < %s -mtriple=amdgcn -mcpu=verde -verify-machineinstrs | FileCheck --check-prefixes=GFX6 %s ;RUN: llc < %s -mtriple=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck --check-prefixes=GFX8PLUS %s ;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs | FileCheck --check-prefixes=GFX11 %s +;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -mattr=-enable-prt-strict-null -verify-machineinstrs | FileCheck --check-prefixes=NOPRT %s ;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs | FileCheck --check-prefixes=GFX12,GFX12-SDAG %s ;RUN: llc < %s -global-isel -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs | FileCheck --check-prefixes=GFX12,GFX12-GISEL %s @@ -34,6 +35,16 @@ define amdgpu_ps {<4 x float>, <4 x float>, <4 x float>} @buffer_load(<4 x i32> ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v8, 0 +; NOPRT-NEXT: s_clause 0x2 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v8, s[0:3], 0 idxen +; NOPRT-NEXT: buffer_load_format_xyzw v[4:7], v8, s[0:3], 0 idxen glc +; NOPRT-NEXT: buffer_load_format_xyzw v[8:11], v8, s[0:3], 0 idxen slc +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_mov_b32_e32 v8, 0 @@ -75,6 +86,13 @@ define amdgpu_ps <4 x float> @buffer_load_immoffs(<4 x i32> inreg) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_immoffs: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v0, 0 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v0, s[0:3], 0 idxen offset:42 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_immoffs: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_mov_b32_e32 v0, 0 @@ -146,6 +164,25 @@ define amdgpu_ps <4 x float> @buffer_load_immoffs_large(<4 x i32> inreg) { ; GFX11-NEXT: v_add_f32_e32 v2, v10, v2 ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_immoffs_large: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v8, 0 +; NOPRT-NEXT: s_movk_i32 s4, 0x7ffc +; NOPRT-NEXT: s_clause 0x1 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v8, s[0:3], 60 idxen offset:4092 +; NOPRT-NEXT: buffer_load_format_xyzw v[4:7], v8, s[0:3], s4 idxen offset:4092 +; NOPRT-NEXT: s_mov_b32 s4, 0x8ffc +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: v_add_f32_e32 v1, v1, v5 +; NOPRT-NEXT: buffer_load_format_xyzw v[8:11], v8, s[0:3], s4 idxen offset:4 +; NOPRT-NEXT: v_dual_add_f32 v0, v0, v4 :: v_dual_add_f32 v3, v3, v7 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: v_dual_add_f32 v2, v2, v6 :: v_dual_add_f32 v1, v9, v1 +; NOPRT-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; NOPRT-NEXT: v_dual_add_f32 v0, v8, v0 :: v_dual_add_f32 v3, v11, v3 +; NOPRT-NEXT: v_add_f32_e32 v2, v10, v2 +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_immoffs_large: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_mov_b32_e32 v8, 0 @@ -196,6 +233,13 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_12bit(<4 x i32> inreg) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_voffset_large_12bit: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v0, 0 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v0, s[0:3], 0 idxen offset:4092 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_voffset_large_12bit: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_mov_b32_e32 v0, 0 @@ -235,6 +279,15 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_13bit(<4 x i32> inreg) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_voffset_large_13bit: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: s_mov_b32 s4, 0 +; NOPRT-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; NOPRT-NEXT: v_dual_mov_b32 v1, 0x1000 :: v_dual_mov_b32 v0, s4 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_voffset_large_13bit: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_mov_b32_e32 v0, 0 @@ -274,6 +327,15 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_16bit(<4 x i32> inreg) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_voffset_large_16bit: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: s_mov_b32 s4, 0 +; NOPRT-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; NOPRT-NEXT: v_dual_mov_b32 v1, 0xf000 :: v_dual_mov_b32 v0, s4 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_voffset_large_16bit: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_mov_b32_e32 v0, 0 @@ -313,6 +375,15 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_23bit(<4 x i32> inreg) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_voffset_large_23bit: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: s_mov_b32 s4, 0 +; NOPRT-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; NOPRT-NEXT: v_dual_mov_b32 v1, 0x7ff000 :: v_dual_mov_b32 v0, s4 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_voffset_large_23bit: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_mov_b32_e32 v0, 0 @@ -352,6 +423,15 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_24bit(<4 x i32> inreg) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_voffset_large_24bit: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: s_mov_b32 s4, 0 +; NOPRT-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; NOPRT-NEXT: v_dual_mov_b32 v1, 0xfff000 :: v_dual_mov_b32 v0, s4 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-SDAG-LABEL: buffer_load_voffset_large_24bit: ; GFX12-SDAG: ; %bb.0: ; %main_body ; GFX12-SDAG-NEXT: v_dual_mov_b32 v1, 0x800000 :: v_dual_mov_b32 v0, 0 @@ -389,6 +469,12 @@ define amdgpu_ps <4 x float> @buffer_load_idx(<4 x i32> inreg, i32) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_idx: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v0, s[0:3], 0 idxen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_idx: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: buffer_load_format_xyzw v[0:3], v0, s[0:3], null idxen @@ -427,6 +513,15 @@ define amdgpu_ps <4 x float> @buffer_load_ofs(<4 x i32> inreg, i32) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_ofs: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: s_mov_b32 s4, 0 +; NOPRT-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; NOPRT-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s4 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_ofs: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, 0 @@ -466,6 +561,15 @@ define amdgpu_ps <4 x float> @buffer_load_ofs_imm(<4 x i32> inreg, i32) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_ofs_imm: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: s_mov_b32 s4, 0 +; NOPRT-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; NOPRT-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s4 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:60 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_ofs_imm: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, 0 @@ -497,6 +601,12 @@ define amdgpu_ps <4 x float> @buffer_load_both(<4 x i32> inreg, i32, i32) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_both: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_both: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], null idxen offen @@ -529,6 +639,13 @@ define amdgpu_ps <4 x float> @buffer_load_both_reversed(<4 x i32> inreg, i32, i3 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_both_reversed: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v2, v0 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[1:2], s[0:3], 0 idxen offen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_both_reversed: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_mov_b32_e32 v2, v0 @@ -562,6 +679,13 @@ define amdgpu_ps float @buffer_load_x(<4 x i32> inreg %rsrc) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_x: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v0, 0 +; NOPRT-NEXT: buffer_load_format_x v0, v0, s[0:3], 0 idxen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_x: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_mov_b32_e32 v0, 0 @@ -595,6 +719,13 @@ define amdgpu_ps float @buffer_load_x_i32(<4 x i32> inreg %rsrc) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_x_i32: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v0, 0 +; NOPRT-NEXT: buffer_load_format_x v0, v0, s[0:3], 0 idxen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_x_i32: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_mov_b32_e32 v0, 0 @@ -629,6 +760,13 @@ define amdgpu_ps <2 x float> @buffer_load_xy(<4 x i32> inreg %rsrc) { ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_xy: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v0, 0 +; NOPRT-NEXT: buffer_load_format_xy v[0:1], v0, s[0:3], 0 idxen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_xy: ; GFX12: ; %bb.0: ; %main_body ; GFX12-NEXT: v_mov_b32_e32 v0, 0 @@ -644,7 +782,12 @@ define amdgpu_cs float @buffer_load_v4i32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX6-LABEL: buffer_load_v4i32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 -; GFX6-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe +; GFX6-NEXT: v_mov_b32_e32 v7, 2 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 +; GFX6-NEXT: v_mov_b32_e32 v4, v2 +; GFX6-NEXT: v_mov_b32_e32 v5, v2 +; GFX6-NEXT: v_mov_b32_e32 v6, v2 +; GFX6-NEXT: buffer_load_format_xyzw v[2:6], v7, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 ; GFX6-NEXT: s_mov_b32 s0, s2 @@ -658,7 +801,12 @@ define amdgpu_cs float @buffer_load_v4i32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX8PLUS-LABEL: buffer_load_v4i32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 -; GFX8PLUS-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe +; GFX8PLUS-NEXT: v_mov_b32_e32 v7, 2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v4, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v5, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v6, v2 +; GFX8PLUS-NEXT: buffer_load_format_xyzw v[2:6], v7, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dwordx4 v[0:1], v[2:5] ; GFX8PLUS-NEXT: v_mov_b32_e32 v0, v6 @@ -667,22 +815,40 @@ define amdgpu_cs float @buffer_load_v4i32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; ; GFX11-LABEL: buffer_load_v4i32_tfe: ; GFX11: ; %bb.0: -; GFX11-NEXT: v_mov_b32_e32 v2, 0 -; GFX11-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe +; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v7, 2 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-NEXT: v_mov_b32_e32 v4, v2 +; GFX11-NEXT: v_mov_b32_e32 v5, v2 +; GFX11-NEXT: v_mov_b32_e32 v6, v2 +; GFX11-NEXT: buffer_load_format_xyzw v[2:6], v7, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b128 v[0:1], v[2:5], off ; GFX11-NEXT: v_mov_b32_e32 v0, v6 ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_v4i32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v2, 2 +; NOPRT-NEXT: v_mov_b32_e32 v6, 0 +; NOPRT-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b128 v[0:1], v[2:5], off +; NOPRT-NEXT: v_mov_b32_e32 v0, v6 +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_v4i32_tfe: ; GFX12: ; %bb.0: -; GFX12-NEXT: v_mov_b32_e32 v2, 0 -; GFX12-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], null idxen tfe +; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v7, 2 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v4, v2 +; GFX12-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v6, v2 +; GFX12-NEXT: buffer_load_format_xyzw v[2:6], v7, s[0:3], null idxen tfe ; GFX12-NEXT: s_wait_loadcnt 0x0 ; GFX12-NEXT: global_store_b128 v[0:1], v[2:5], off ; GFX12-NEXT: v_mov_b32_e32 v0, v6 ; GFX12-NEXT: ; return to shader part epilog - %load = call { <4 x i32>, i32 } @llvm.amdgcn.struct.buffer.load.format.sl_v4i32i32s(<4 x i32> %rsrc, i32 0, i32 0, i32 0, i32 0) + %load = call { <4 x i32>, i32 } @llvm.amdgcn.struct.buffer.load.format.sl_v4i32i32s(<4 x i32> %rsrc, i32 2, i32 0, i32 0, i32 0) %data = extractvalue { <4 x i32>, i32 } %load, 0 store <4 x i32> %data, ptr addrspace(1) %out %status = extractvalue { <4 x i32>, i32 } %load, 1 @@ -694,6 +860,10 @@ define amdgpu_cs float @buffer_load_v4f32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX6-LABEL: buffer_load_v4f32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 +; GFX6-NEXT: v_mov_b32_e32 v4, v2 +; GFX6-NEXT: v_mov_b32_e32 v5, v2 +; GFX6-NEXT: v_mov_b32_e32 v6, v2 ; GFX6-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -708,6 +878,10 @@ define amdgpu_cs float @buffer_load_v4f32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX8PLUS-LABEL: buffer_load_v4f32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v4, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v5, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v6, v2 ; GFX8PLUS-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dwordx4 v[0:1], v[2:5] @@ -718,15 +892,32 @@ define amdgpu_cs float @buffer_load_v4f32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX11-LABEL: buffer_load_v4f32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-NEXT: v_mov_b32_e32 v4, v2 +; GFX11-NEXT: v_mov_b32_e32 v5, v2 +; GFX11-NEXT: v_mov_b32_e32 v6, v2 ; GFX11-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b128 v[0:1], v[2:5], off ; GFX11-NEXT: v_mov_b32_e32 v0, v6 ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_v4f32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v6, 0 +; NOPRT-NEXT: buffer_load_format_xyzw v[2:6], v6, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b128 v[0:1], v[2:5], off +; NOPRT-NEXT: v_mov_b32_e32 v0, v6 +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_v4f32_tfe: ; GFX12: ; %bb.0: ; GFX12-NEXT: v_mov_b32_e32 v2, 0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v4, v2 +; GFX12-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v6, v2 ; GFX12-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], null idxen tfe ; GFX12-NEXT: s_wait_loadcnt 0x0 ; GFX12-NEXT: global_store_b128 v[0:1], v[2:5], off @@ -744,6 +935,9 @@ define amdgpu_cs float @buffer_load_v3i32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX6-LABEL: buffer_load_v3i32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 +; GFX6-NEXT: v_mov_b32_e32 v4, v2 +; GFX6-NEXT: v_mov_b32_e32 v5, v2 ; GFX6-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -759,6 +953,9 @@ define amdgpu_cs float @buffer_load_v3i32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX8PLUS-LABEL: buffer_load_v3i32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v4, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v5, v2 ; GFX8PLUS-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dwordx3 v[0:1], v[2:4] @@ -769,15 +966,31 @@ define amdgpu_cs float @buffer_load_v3i32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX11-LABEL: buffer_load_v3i32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-NEXT: v_mov_b32_e32 v4, v2 +; GFX11-NEXT: v_mov_b32_e32 v5, v2 ; GFX11-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b96 v[0:1], v[2:4], off ; GFX11-NEXT: v_mov_b32_e32 v0, v5 ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_v3i32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v5, 0 +; NOPRT-NEXT: buffer_load_format_xyz v[2:5], v5, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b96 v[0:1], v[2:4], off +; NOPRT-NEXT: v_mov_b32_e32 v0, v5 +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_v3i32_tfe: ; GFX12: ; %bb.0: ; GFX12-NEXT: v_mov_b32_e32 v2, 0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v4, v2 +; GFX12-NEXT: v_mov_b32_e32 v5, v2 ; GFX12-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], null idxen tfe ; GFX12-NEXT: s_wait_loadcnt 0x0 ; GFX12-NEXT: global_store_b96 v[0:1], v[2:4], off @@ -795,6 +1008,9 @@ define amdgpu_cs float @buffer_load_v3f32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX6-LABEL: buffer_load_v3f32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 +; GFX6-NEXT: v_mov_b32_e32 v4, v2 +; GFX6-NEXT: v_mov_b32_e32 v5, v2 ; GFX6-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -810,6 +1026,9 @@ define amdgpu_cs float @buffer_load_v3f32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX8PLUS-LABEL: buffer_load_v3f32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v4, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v5, v2 ; GFX8PLUS-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dwordx3 v[0:1], v[2:4] @@ -820,15 +1039,31 @@ define amdgpu_cs float @buffer_load_v3f32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX11-LABEL: buffer_load_v3f32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-NEXT: v_mov_b32_e32 v4, v2 +; GFX11-NEXT: v_mov_b32_e32 v5, v2 ; GFX11-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b96 v[0:1], v[2:4], off ; GFX11-NEXT: v_mov_b32_e32 v0, v5 ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_v3f32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v5, 0 +; NOPRT-NEXT: buffer_load_format_xyz v[2:5], v5, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b96 v[0:1], v[2:4], off +; NOPRT-NEXT: v_mov_b32_e32 v0, v5 +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_v3f32_tfe: ; GFX12: ; %bb.0: ; GFX12-NEXT: v_mov_b32_e32 v2, 0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v4, v2 +; GFX12-NEXT: v_mov_b32_e32 v5, v2 ; GFX12-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], null idxen tfe ; GFX12-NEXT: s_wait_loadcnt 0x0 ; GFX12-NEXT: global_store_b96 v[0:1], v[2:4], off @@ -846,6 +1081,9 @@ define amdgpu_cs float @buffer_load_v2i32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX6-LABEL: buffer_load_v2i32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 +; GFX6-NEXT: v_mov_b32_e32 v4, v2 +; GFX6-NEXT: v_mov_b32_e32 v5, v2 ; GFX6-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -860,6 +1098,8 @@ define amdgpu_cs float @buffer_load_v2i32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX8PLUS-LABEL: buffer_load_v2i32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v4, v2 ; GFX8PLUS-NEXT: buffer_load_format_xy v[2:4], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dwordx2 v[0:1], v[2:3] @@ -870,15 +1110,29 @@ define amdgpu_cs float @buffer_load_v2i32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX11-LABEL: buffer_load_v2i32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-NEXT: v_mov_b32_e32 v4, v2 ; GFX11-NEXT: buffer_load_format_xy v[2:4], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b64 v[0:1], v[2:3], off ; GFX11-NEXT: v_mov_b32_e32 v0, v4 ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_v2i32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v4, 0 +; NOPRT-NEXT: buffer_load_format_xy v[2:4], v4, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b64 v[0:1], v[2:3], off +; NOPRT-NEXT: v_mov_b32_e32 v0, v4 +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_v2i32_tfe: ; GFX12: ; %bb.0: ; GFX12-NEXT: v_mov_b32_e32 v2, 0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v4, v2 ; GFX12-NEXT: buffer_load_format_xy v[2:4], v2, s[0:3], null idxen tfe ; GFX12-NEXT: s_wait_loadcnt 0x0 ; GFX12-NEXT: global_store_b64 v[0:1], v[2:3], off @@ -896,6 +1150,9 @@ define amdgpu_cs float @buffer_load_v2f32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX6-LABEL: buffer_load_v2f32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 +; GFX6-NEXT: v_mov_b32_e32 v4, v2 +; GFX6-NEXT: v_mov_b32_e32 v5, v2 ; GFX6-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -910,6 +1167,8 @@ define amdgpu_cs float @buffer_load_v2f32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX8PLUS-LABEL: buffer_load_v2f32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v4, v2 ; GFX8PLUS-NEXT: buffer_load_format_xy v[2:4], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dwordx2 v[0:1], v[2:3] @@ -920,15 +1179,29 @@ define amdgpu_cs float @buffer_load_v2f32_tfe(<4 x i32> inreg %rsrc, ptr addrspa ; GFX11-LABEL: buffer_load_v2f32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-NEXT: v_mov_b32_e32 v4, v2 ; GFX11-NEXT: buffer_load_format_xy v[2:4], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b64 v[0:1], v[2:3], off ; GFX11-NEXT: v_mov_b32_e32 v0, v4 ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_v2f32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v4, 0 +; NOPRT-NEXT: buffer_load_format_xy v[2:4], v4, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b64 v[0:1], v[2:3], off +; NOPRT-NEXT: v_mov_b32_e32 v0, v4 +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_v2f32_tfe: ; GFX12: ; %bb.0: ; GFX12-NEXT: v_mov_b32_e32 v2, 0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v4, v2 ; GFX12-NEXT: buffer_load_format_xy v[2:4], v2, s[0:3], null idxen tfe ; GFX12-NEXT: s_wait_loadcnt 0x0 ; GFX12-NEXT: global_store_b64 v[0:1], v[2:3], off @@ -946,6 +1219,7 @@ define amdgpu_cs float @buffer_load_i32_tfe(<4 x i32> inreg %rsrc, ptr addrspace ; GFX6-LABEL: buffer_load_i32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 ; GFX6-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -960,6 +1234,7 @@ define amdgpu_cs float @buffer_load_i32_tfe(<4 x i32> inreg %rsrc, ptr addrspace ; GFX8PLUS-LABEL: buffer_load_i32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 ; GFX8PLUS-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dword v[0:1], v2 @@ -970,15 +1245,28 @@ define amdgpu_cs float @buffer_load_i32_tfe(<4 x i32> inreg %rsrc, ptr addrspace ; GFX11-LABEL: buffer_load_i32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 ; GFX11-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b32 v[0:1], v2, off ; GFX11-NEXT: v_mov_b32_e32 v0, v3 ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_i32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v3, 0 +; NOPRT-NEXT: buffer_load_format_x v[2:3], v3, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b32 v[0:1], v2, off +; NOPRT-NEXT: v_mov_b32_e32 v0, v3 +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_i32_tfe: ; GFX12: ; %bb.0: ; GFX12-NEXT: v_mov_b32_e32 v2, 0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_mov_b32_e32 v3, v2 ; GFX12-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], null idxen tfe ; GFX12-NEXT: s_wait_loadcnt 0x0 ; GFX12-NEXT: global_store_b32 v[0:1], v2, off @@ -996,6 +1284,7 @@ define amdgpu_cs float @buffer_load_f32_tfe(<4 x i32> inreg %rsrc, ptr addrspace ; GFX6-LABEL: buffer_load_f32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 ; GFX6-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -1010,6 +1299,7 @@ define amdgpu_cs float @buffer_load_f32_tfe(<4 x i32> inreg %rsrc, ptr addrspace ; GFX8PLUS-LABEL: buffer_load_f32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 ; GFX8PLUS-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dword v[0:1], v2 @@ -1020,15 +1310,28 @@ define amdgpu_cs float @buffer_load_f32_tfe(<4 x i32> inreg %rsrc, ptr addrspace ; GFX11-LABEL: buffer_load_f32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 ; GFX11-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b32 v[0:1], v2, off ; GFX11-NEXT: v_mov_b32_e32 v0, v3 ; GFX11-NEXT: ; return to shader part epilog ; +; NOPRT-LABEL: buffer_load_f32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v3, 0 +; NOPRT-NEXT: buffer_load_format_x v[2:3], v3, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b32 v[0:1], v2, off +; NOPRT-NEXT: v_mov_b32_e32 v0, v3 +; NOPRT-NEXT: ; return to shader part epilog +; ; GFX12-LABEL: buffer_load_f32_tfe: ; GFX12: ; %bb.0: ; GFX12-NEXT: v_mov_b32_e32 v2, 0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_mov_b32_e32 v3, v2 ; GFX12-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], null idxen tfe ; GFX12-NEXT: s_wait_loadcnt 0x0 ; GFX12-NEXT: global_store_b32 v[0:1], v2, off diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.ll index b0bd4e428ef2..c5202b84fa1e 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.ll @@ -2,6 +2,7 @@ ;RUN: llc < %s -mtriple=amdgcn -mcpu=verde -verify-machineinstrs | FileCheck --check-prefixes=GFX6 %s ;RUN: llc < %s -mtriple=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck --check-prefixes=GFX8PLUS %s ;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs | FileCheck --check-prefixes=GFX11 %s +;RUN: llc < %s -mtriple=amdgcn -mattr=-enable-prt-strict-null -mcpu=gfx1100 -verify-machineinstrs | FileCheck --check-prefixes=NOPRT %s define amdgpu_ps {<4 x float>, <4 x float>, <4 x float>} @buffer_load(ptr addrspace(8) inreg) { ; GFX6-LABEL: buffer_load: @@ -31,6 +32,16 @@ define amdgpu_ps {<4 x float>, <4 x float>, <4 x float>} @buffer_load(ptr addrsp ; GFX11-NEXT: buffer_load_format_xyzw v[8:11], v8, s[0:3], 0 idxen slc ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v8, 0 +; NOPRT-NEXT: s_clause 0x2 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v8, s[0:3], 0 idxen +; NOPRT-NEXT: buffer_load_format_xyzw v[4:7], v8, s[0:3], 0 idxen glc +; NOPRT-NEXT: buffer_load_format_xyzw v[8:11], v8, s[0:3], 0 idxen slc +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 0, i32 0, i32 0) %data_glc = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 0, i32 0, i32 1) @@ -62,6 +73,13 @@ define amdgpu_ps <4 x float> @buffer_load_immoffs(ptr addrspace(8) inreg) { ; GFX11-NEXT: buffer_load_format_xyzw v[0:3], v0, s[0:3], 0 idxen offset:42 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_immoffs: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v0, 0 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v0, s[0:3], 0 idxen offset:42 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 42, i32 0, i32 0) ret <4 x float> %data @@ -126,6 +144,25 @@ define amdgpu_ps <4 x float> @buffer_load_immoffs_large(ptr addrspace(8) inreg) ; GFX11-NEXT: v_dual_add_f32 v0, v8, v0 :: v_dual_add_f32 v3, v11, v3 ; GFX11-NEXT: v_add_f32_e32 v2, v10, v2 ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_immoffs_large: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v8, 0 +; NOPRT-NEXT: s_movk_i32 s4, 0x7ffc +; NOPRT-NEXT: s_clause 0x1 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v8, s[0:3], 60 idxen offset:4092 +; NOPRT-NEXT: buffer_load_format_xyzw v[4:7], v8, s[0:3], s4 idxen offset:4092 +; NOPRT-NEXT: s_mov_b32 s4, 0x8ffc +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: v_add_f32_e32 v1, v1, v5 +; NOPRT-NEXT: buffer_load_format_xyzw v[8:11], v8, s[0:3], s4 idxen offset:4 +; NOPRT-NEXT: v_dual_add_f32 v0, v0, v4 :: v_dual_add_f32 v3, v3, v7 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: v_dual_add_f32 v2, v2, v6 :: v_dual_add_f32 v1, v9, v1 +; NOPRT-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; NOPRT-NEXT: v_dual_add_f32 v0, v8, v0 :: v_dual_add_f32 v3, v11, v3 +; NOPRT-NEXT: v_add_f32_e32 v2, v10, v2 +; NOPRT-NEXT: ; return to shader part epilog main_body: %d.0 = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 4092, i32 60, i32 0) %d.1 = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 4092, i32 32764, i32 0) @@ -156,6 +193,13 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_12bit(ptr addrspace(8) i ; GFX11-NEXT: buffer_load_format_xyzw v[0:3], v0, s[0:3], 0 idxen offset:4092 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_voffset_large_12bit: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v0, 0 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v0, s[0:3], 0 idxen offset:4092 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 4092, i32 0, i32 0) ret <4 x float> %data @@ -188,6 +232,15 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_13bit(ptr addrspace(8) i ; GFX11-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_voffset_large_13bit: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: s_mov_b32 s4, 0 +; NOPRT-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; NOPRT-NEXT: v_dual_mov_b32 v1, 0x1000 :: v_dual_mov_b32 v0, s4 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 8188, i32 0, i32 0) ret <4 x float> %data @@ -220,6 +273,15 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_16bit(ptr addrspace(8) i ; GFX11-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_voffset_large_16bit: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: s_mov_b32 s4, 0 +; NOPRT-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; NOPRT-NEXT: v_dual_mov_b32 v1, 0xf000 :: v_dual_mov_b32 v0, s4 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 65532, i32 0, i32 0) ret <4 x float> %data @@ -252,6 +314,15 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_23bit(ptr addrspace(8) i ; GFX11-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_voffset_large_23bit: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: s_mov_b32 s4, 0 +; NOPRT-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; NOPRT-NEXT: v_dual_mov_b32 v1, 0x7ff000 :: v_dual_mov_b32 v0, s4 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 8388604, i32 0, i32 0) ret <4 x float> %data @@ -284,6 +355,15 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_24bit(ptr addrspace(8) i ; GFX11-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_voffset_large_24bit: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: s_mov_b32 s4, 0 +; NOPRT-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; NOPRT-NEXT: v_dual_mov_b32 v1, 0xfff000 :: v_dual_mov_b32 v0, s4 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 16777212, i32 0, i32 0) ret <4 x float> %data @@ -307,6 +387,12 @@ define amdgpu_ps <4 x float> @buffer_load_idx(ptr addrspace(8) inreg, i32) { ; GFX11-NEXT: buffer_load_format_xyzw v[0:3], v0, s[0:3], 0 idxen ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_idx: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v0, s[0:3], 0 idxen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 %1, i32 0, i32 0, i32 0) ret <4 x float> %data @@ -339,6 +425,15 @@ define amdgpu_ps <4 x float> @buffer_load_ofs(ptr addrspace(8) inreg, i32) { ; GFX11-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_ofs: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: s_mov_b32 s4, 0 +; NOPRT-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; NOPRT-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s4 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 %1, i32 0, i32 0) ret <4 x float> %data @@ -371,6 +466,15 @@ define amdgpu_ps <4 x float> @buffer_load_ofs_imm(ptr addrspace(8) inreg, i32) { ; GFX11-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:60 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_ofs_imm: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: s_mov_b32 s4, 0 +; NOPRT-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; NOPRT-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s4 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:60 +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %ofs = add i32 %1, 60 %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 %ofs, i32 0, i32 0) @@ -395,6 +499,12 @@ define amdgpu_ps <4 x float> @buffer_load_both(ptr addrspace(8) inreg, i32, i32) ; GFX11-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_both: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 %1, i32 %2, i32 0, i32 0) ret <4 x float> %data @@ -421,6 +531,13 @@ define amdgpu_ps <4 x float> @buffer_load_both_reversed(ptr addrspace(8) inreg, ; GFX11-NEXT: buffer_load_format_xyzw v[0:3], v[1:2], s[0:3], 0 idxen offen ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_both_reversed: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v2, v0 +; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v[1:2], s[0:3], 0 idxen offen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 %2, i32 %1, i32 0, i32 0) ret <4 x float> %data @@ -447,6 +564,13 @@ define amdgpu_ps float @buffer_load_x(ptr addrspace(8) inreg %rsrc) { ; GFX11-NEXT: buffer_load_format_x v0, v0, s[0:3], 0 idxen ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_x: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v0, 0 +; NOPRT-NEXT: buffer_load_format_x v0, v0, s[0:3], 0 idxen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call float @llvm.amdgcn.struct.ptr.buffer.load.format.f32(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0) ret float %data @@ -473,6 +597,13 @@ define amdgpu_ps float @buffer_load_x_i32(ptr addrspace(8) inreg %rsrc) { ; GFX11-NEXT: buffer_load_format_x v0, v0, s[0:3], 0 idxen ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_x_i32: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v0, 0 +; NOPRT-NEXT: buffer_load_format_x v0, v0, s[0:3], 0 idxen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call i32 @llvm.amdgcn.struct.ptr.buffer.load.format.i32(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0) %fdata = bitcast i32 %data to float @@ -500,6 +631,13 @@ define amdgpu_ps <2 x float> @buffer_load_xy(ptr addrspace(8) inreg %rsrc) { ; GFX11-NEXT: buffer_load_format_xy v[0:1], v0, s[0:3], 0 idxen ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_xy: +; NOPRT: ; %bb.0: ; %main_body +; NOPRT-NEXT: v_mov_b32_e32 v0, 0 +; NOPRT-NEXT: buffer_load_format_xy v[0:1], v0, s[0:3], 0 idxen +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: ; return to shader part epilog main_body: %data = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v2f32(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0) ret <2 x float> %data @@ -509,6 +647,10 @@ define amdgpu_cs float @buffer_load_v4i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX6-LABEL: buffer_load_v4i32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 +; GFX6-NEXT: v_mov_b32_e32 v4, v2 +; GFX6-NEXT: v_mov_b32_e32 v5, v2 +; GFX6-NEXT: v_mov_b32_e32 v6, v2 ; GFX6-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -523,6 +665,10 @@ define amdgpu_cs float @buffer_load_v4i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX8PLUS-LABEL: buffer_load_v4i32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v4, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v5, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v6, v2 ; GFX8PLUS-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dwordx4 v[0:1], v[2:5] @@ -533,11 +679,25 @@ define amdgpu_cs float @buffer_load_v4i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX11-LABEL: buffer_load_v4i32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-NEXT: v_mov_b32_e32 v4, v2 +; GFX11-NEXT: v_mov_b32_e32 v5, v2 +; GFX11-NEXT: v_mov_b32_e32 v6, v2 ; GFX11-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b128 v[0:1], v[2:5], off ; GFX11-NEXT: v_mov_b32_e32 v0, v6 ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_v4i32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v6, 0 +; NOPRT-NEXT: buffer_load_format_xyzw v[2:6], v6, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b128 v[0:1], v[2:5], off +; NOPRT-NEXT: v_mov_b32_e32 v0, v6 +; NOPRT-NEXT: ; return to shader part epilog %load = call { <4 x i32>, i32 } @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v4i32i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0) %data = extractvalue { <4 x i32>, i32 } %load, 0 store <4 x i32> %data, ptr addrspace(1) %out @@ -550,6 +710,10 @@ define amdgpu_cs float @buffer_load_v4f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX6-LABEL: buffer_load_v4f32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 +; GFX6-NEXT: v_mov_b32_e32 v4, v2 +; GFX6-NEXT: v_mov_b32_e32 v5, v2 +; GFX6-NEXT: v_mov_b32_e32 v6, v2 ; GFX6-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -564,6 +728,10 @@ define amdgpu_cs float @buffer_load_v4f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX8PLUS-LABEL: buffer_load_v4f32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v4, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v5, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v6, v2 ; GFX8PLUS-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dwordx4 v[0:1], v[2:5] @@ -574,11 +742,25 @@ define amdgpu_cs float @buffer_load_v4f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX11-LABEL: buffer_load_v4f32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-NEXT: v_mov_b32_e32 v4, v2 +; GFX11-NEXT: v_mov_b32_e32 v5, v2 +; GFX11-NEXT: v_mov_b32_e32 v6, v2 ; GFX11-NEXT: buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b128 v[0:1], v[2:5], off ; GFX11-NEXT: v_mov_b32_e32 v0, v6 ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_v4f32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v6, 0 +; NOPRT-NEXT: buffer_load_format_xyzw v[2:6], v6, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b128 v[0:1], v[2:5], off +; NOPRT-NEXT: v_mov_b32_e32 v0, v6 +; NOPRT-NEXT: ; return to shader part epilog %load = call { <4 x float>, i32 } @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v4f32i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0) %data = extractvalue { <4 x float>, i32 } %load, 0 store <4 x float> %data, ptr addrspace(1) %out @@ -591,6 +773,9 @@ define amdgpu_cs float @buffer_load_v3i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX6-LABEL: buffer_load_v3i32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 +; GFX6-NEXT: v_mov_b32_e32 v4, v2 +; GFX6-NEXT: v_mov_b32_e32 v5, v2 ; GFX6-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -606,6 +791,9 @@ define amdgpu_cs float @buffer_load_v3i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX8PLUS-LABEL: buffer_load_v3i32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v4, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v5, v2 ; GFX8PLUS-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dwordx3 v[0:1], v[2:4] @@ -616,11 +804,24 @@ define amdgpu_cs float @buffer_load_v3i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX11-LABEL: buffer_load_v3i32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-NEXT: v_mov_b32_e32 v4, v2 +; GFX11-NEXT: v_mov_b32_e32 v5, v2 ; GFX11-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b96 v[0:1], v[2:4], off ; GFX11-NEXT: v_mov_b32_e32 v0, v5 ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_v3i32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v5, 0 +; NOPRT-NEXT: buffer_load_format_xyz v[2:5], v5, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b96 v[0:1], v[2:4], off +; NOPRT-NEXT: v_mov_b32_e32 v0, v5 +; NOPRT-NEXT: ; return to shader part epilog %load = call { <3 x i32>, i32 } @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v3i32i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0) %data = extractvalue { <3 x i32>, i32 } %load, 0 store <3 x i32> %data, ptr addrspace(1) %out @@ -633,6 +834,9 @@ define amdgpu_cs float @buffer_load_v3f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX6-LABEL: buffer_load_v3f32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 +; GFX6-NEXT: v_mov_b32_e32 v4, v2 +; GFX6-NEXT: v_mov_b32_e32 v5, v2 ; GFX6-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -648,6 +852,9 @@ define amdgpu_cs float @buffer_load_v3f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX8PLUS-LABEL: buffer_load_v3f32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v4, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v5, v2 ; GFX8PLUS-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dwordx3 v[0:1], v[2:4] @@ -658,11 +865,24 @@ define amdgpu_cs float @buffer_load_v3f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX11-LABEL: buffer_load_v3f32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-NEXT: v_mov_b32_e32 v4, v2 +; GFX11-NEXT: v_mov_b32_e32 v5, v2 ; GFX11-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b96 v[0:1], v[2:4], off ; GFX11-NEXT: v_mov_b32_e32 v0, v5 ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_v3f32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v5, 0 +; NOPRT-NEXT: buffer_load_format_xyz v[2:5], v5, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b96 v[0:1], v[2:4], off +; NOPRT-NEXT: v_mov_b32_e32 v0, v5 +; NOPRT-NEXT: ; return to shader part epilog %load = call { <3 x float>, i32 } @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v3f32i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0) %data = extractvalue { <3 x float>, i32 } %load, 0 store <3 x float> %data, ptr addrspace(1) %out @@ -675,6 +895,9 @@ define amdgpu_cs float @buffer_load_v2i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX6-LABEL: buffer_load_v2i32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 +; GFX6-NEXT: v_mov_b32_e32 v4, v2 +; GFX6-NEXT: v_mov_b32_e32 v5, v2 ; GFX6-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -689,6 +912,8 @@ define amdgpu_cs float @buffer_load_v2i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX8PLUS-LABEL: buffer_load_v2i32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v4, v2 ; GFX8PLUS-NEXT: buffer_load_format_xy v[2:4], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dwordx2 v[0:1], v[2:3] @@ -699,11 +924,23 @@ define amdgpu_cs float @buffer_load_v2i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX11-LABEL: buffer_load_v2i32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-NEXT: v_mov_b32_e32 v4, v2 ; GFX11-NEXT: buffer_load_format_xy v[2:4], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b64 v[0:1], v[2:3], off ; GFX11-NEXT: v_mov_b32_e32 v0, v4 ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_v2i32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v4, 0 +; NOPRT-NEXT: buffer_load_format_xy v[2:4], v4, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b64 v[0:1], v[2:3], off +; NOPRT-NEXT: v_mov_b32_e32 v0, v4 +; NOPRT-NEXT: ; return to shader part epilog %load = call { <2 x i32>, i32 } @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v2i32i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0) %data = extractvalue { <2 x i32>, i32 } %load, 0 store <2 x i32> %data, ptr addrspace(1) %out @@ -716,6 +953,9 @@ define amdgpu_cs float @buffer_load_v2f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX6-LABEL: buffer_load_v2f32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 +; GFX6-NEXT: v_mov_b32_e32 v4, v2 +; GFX6-NEXT: v_mov_b32_e32 v5, v2 ; GFX6-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -730,6 +970,8 @@ define amdgpu_cs float @buffer_load_v2f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX8PLUS-LABEL: buffer_load_v2f32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 +; GFX8PLUS-NEXT: v_mov_b32_e32 v4, v2 ; GFX8PLUS-NEXT: buffer_load_format_xy v[2:4], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dwordx2 v[0:1], v[2:3] @@ -740,11 +982,23 @@ define amdgpu_cs float @buffer_load_v2f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ; GFX11-LABEL: buffer_load_v2f32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-NEXT: v_mov_b32_e32 v4, v2 ; GFX11-NEXT: buffer_load_format_xy v[2:4], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b64 v[0:1], v[2:3], off ; GFX11-NEXT: v_mov_b32_e32 v0, v4 ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_v2f32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v4, 0 +; NOPRT-NEXT: buffer_load_format_xy v[2:4], v4, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b64 v[0:1], v[2:3], off +; NOPRT-NEXT: v_mov_b32_e32 v0, v4 +; NOPRT-NEXT: ; return to shader part epilog %load = call { <2 x float>, i32 } @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v2f32i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0) %data = extractvalue { <2 x float>, i32 } %load, 0 store <2 x float> %data, ptr addrspace(1) %out @@ -757,6 +1011,7 @@ define amdgpu_cs float @buffer_load_i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ad ; GFX6-LABEL: buffer_load_i32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 ; GFX6-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -771,6 +1026,7 @@ define amdgpu_cs float @buffer_load_i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ad ; GFX8PLUS-LABEL: buffer_load_i32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 ; GFX8PLUS-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dword v[0:1], v2 @@ -781,11 +1037,22 @@ define amdgpu_cs float @buffer_load_i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ad ; GFX11-LABEL: buffer_load_i32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 ; GFX11-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b32 v[0:1], v2, off ; GFX11-NEXT: v_mov_b32_e32 v0, v3 ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_i32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v3, 0 +; NOPRT-NEXT: buffer_load_format_x v[2:3], v3, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b32 v[0:1], v2, off +; NOPRT-NEXT: v_mov_b32_e32 v0, v3 +; NOPRT-NEXT: ; return to shader part epilog %load = call { i32, i32 } @llvm.amdgcn.struct.ptr.buffer.load.format.sl_i32i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0) %data = extractvalue { i32, i32 } %load, 0 store i32 %data, ptr addrspace(1) %out @@ -798,6 +1065,7 @@ define amdgpu_cs float @buffer_load_f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ad ; GFX6-LABEL: buffer_load_f32_tfe: ; GFX6: ; %bb.0: ; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: v_mov_b32_e32 v3, v2 ; GFX6-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe ; GFX6-NEXT: s_mov_b32 s2, 0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 @@ -812,6 +1080,7 @@ define amdgpu_cs float @buffer_load_f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ad ; GFX8PLUS-LABEL: buffer_load_f32_tfe: ; GFX8PLUS: ; %bb.0: ; GFX8PLUS-NEXT: v_mov_b32_e32 v2, 0 +; GFX8PLUS-NEXT: v_mov_b32_e32 v3, v2 ; GFX8PLUS-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe ; GFX8PLUS-NEXT: s_waitcnt vmcnt(0) ; GFX8PLUS-NEXT: flat_store_dword v[0:1], v2 @@ -822,11 +1091,22 @@ define amdgpu_cs float @buffer_load_f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ad ; GFX11-LABEL: buffer_load_f32_tfe: ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v3, v2 ; GFX11-NEXT: buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b32 v[0:1], v2, off ; GFX11-NEXT: v_mov_b32_e32 v0, v3 ; GFX11-NEXT: ; return to shader part epilog +; +; NOPRT-LABEL: buffer_load_f32_tfe: +; NOPRT: ; %bb.0: +; NOPRT-NEXT: v_mov_b32_e32 v3, 0 +; NOPRT-NEXT: buffer_load_format_x v[2:3], v3, s[0:3], 0 idxen tfe +; NOPRT-NEXT: s_waitcnt vmcnt(0) +; NOPRT-NEXT: global_store_b32 v[0:1], v2, off +; NOPRT-NEXT: v_mov_b32_e32 v0, v3 +; NOPRT-NEXT: ; return to shader part epilog %load = call { float, i32 } @llvm.amdgcn.struct.ptr.buffer.load.format.sl_f32i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0) %data = extractvalue { float, i32 } %load, 0 store float %data, ptr addrspace(1) %out -- GitLab From 99c40f6ba60ea6e5dbd3f015956c7d2f6e25e54c Mon Sep 17 00:00:00 2001 From: Vyacheslav Levytskyy Date: Mon, 25 Mar 2024 10:13:42 +0100 Subject: [PATCH 190/306] [SPIR-V] Introduce a command line option to support compatibility with Khronos SPIRV Translator (#86101) SPIRV-LLVM-Translator project (https://github.com/KhronosGroup/SPIRV-LLVM-Translator) from Khronos Group is a tool and a library for bi-directional translation between SPIR-V and LLVM IR. In its backward translation from SPIR-V to LLVM IR SPIRV-LLVM-Translator isn't necessarily able to cover the same SPIR-V patterns/instructions set that SPIRV Backend produces, even if we target the same SPIR-V version in both SPIRV-LLVM-Translator and SPIRV Backend projects. To improve interoperability and ability to apply SPIRV Backend output in different products this PR introduces a notion of a mode of SPIR-V output that is compatible with a subset of SPIR-V supported by SPIRV-LLVM-Translator. This includes a new command line option that doesn't influence default behavior of SPIRV Backend and one test case that demonstrates how this command line option may be used to get a practical benefit of producing that one of two possible and similar output options that can be understood by SPIRV-LLVM-Translator. --- llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp | 8 +++++++- llvm/test/CodeGen/SPIRV/instructions/ptrcmp.ll | 12 ++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp b/llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp index 38caa7c8ea0a..09a029a35a74 100644 --- a/llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp @@ -27,6 +27,11 @@ using namespace llvm; #define GET_SUBTARGETINFO_CTOR #include "SPIRVGenSubtargetInfo.inc" +static cl::opt + SPVTranslatorCompat("translator-compatibility-mode", + cl::desc("SPIR-V Translator compatibility mode"), + cl::Optional, cl::init(false)); + cl::list Extensions( "spirv-extensions", cl::desc("SPIR-V extensions"), cl::ZeroOrMore, cl::Hidden, @@ -157,8 +162,9 @@ bool SPIRVSubtarget::isAtLeastOpenCLVer(uint32_t VerToCompareTo) const { } // If the SPIR-V version is >= 1.4 we can call OpPtrEqual and OpPtrNotEqual. +// In SPIR-V Translator compatibility mode this feature is not available. bool SPIRVSubtarget::canDirectlyComparePointers() const { - return isAtLeastVer(SPIRVVersion, 14); + return !SPVTranslatorCompat && isAtLeastVer(SPIRVVersion, 14); } void SPIRVSubtarget::initAvailableExtensions() { diff --git a/llvm/test/CodeGen/SPIRV/instructions/ptrcmp.ll b/llvm/test/CodeGen/SPIRV/instructions/ptrcmp.ll index 641e2bf0649c..31cd8bd45929 100644 --- a/llvm/test/CodeGen/SPIRV/instructions/ptrcmp.ll +++ b/llvm/test/CodeGen/SPIRV/instructions/ptrcmp.ll @@ -1,7 +1,13 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} + +; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s --translator-compatibility-mode -o - | FileCheck %s --check-prefix=CHECK-COMPAT +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s --translator-compatibility-mode -o - -filetype=obj | spirv-val %} ; CHECK-DAG: OpName [[EQ:%.*]] "test_eq" ; CHECK-DAG: OpName [[NE:%.*]] "test_ne" +; CHECK-COMPAT-DAG: OpName [[EQ:%.*]] "test_eq" +; CHECK-COMPAT-DAG: OpName [[NE:%.*]] "test_ne" ; CHECK-DAG: OpName [[ULT:%.*]] "test_ult" ; CHECK-DAG: OpName [[SLT:%.*]] "test_slt" ; CHECK-DAG: OpName [[ULE:%.*]] "test_ule" @@ -19,6 +25,9 @@ ; CHECK-NEXT: [[R:%.*]] = OpPtrEqual {{%.+}} [[A]] [[B]] ; CHECK-NEXT: OpReturnValue [[R]] ; CHECK-NEXT: OpFunctionEnd +; CHECK-COMPAT: [[EQ]] = OpFunction +; CHECK-COMPAT-NOT: OpPtrEqual +; CHECK-COMPAT: OpFunctionEnd define i1 @test_eq(i16* %a, i16* %b) { %r = icmp eq i16* %a, %b ret i1 %r @@ -31,6 +40,9 @@ define i1 @test_eq(i16* %a, i16* %b) { ; CHECK-NEXT: [[R:%.*]] = OpPtrNotEqual {{%.+}} [[A]] [[B]] ; CHECK-NEXT: OpReturnValue [[R]] ; CHECK-NEXT: OpFunctionEnd +; CHECK-COMPAT: [[NE]] = OpFunction +; CHECK-COMPAT-NOT: OpPtrNotEqual +; CHECK-COMPAT: OpFunctionEnd define i1 @test_ne(i16* %a, i16* %b) { %r = icmp ne i16* %a, %b ret i1 %r -- GitLab From 1d250d9099a9ba8b53add7eb7db6827e8fc0c8fd Mon Sep 17 00:00:00 2001 From: Vyacheslav Levytskyy Date: Mon, 25 Mar 2024 10:14:08 +0100 Subject: [PATCH 191/306] [SPIR-V] Improve type inference in SPIR-V Backend for opaque pointers (#86283) This PR improves type inference in SPIR-V Backend for opaque pointers, accounting or a case when there is a chain of function calls that allows to deduce formal parameter types from actual arguments. The attached test demonstrates the case. --- llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 123 +++++++++++------- llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp | 1 + .../Target/SPIRV/SPIRVInstructionSelector.cpp | 12 +- .../pointers/type-deduce-by-call-chain.ll | 52 ++++++++ 4 files changed, 138 insertions(+), 50 deletions(-) create mode 100644 llvm/test/CodeGen/SPIRV/pointers/type-deduce-by-call-chain.ll diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp index 458af9229ed7..5828db6669ff 100644 --- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp @@ -92,6 +92,9 @@ class SPIRVEmitIntrinsics void insertPtrCastOrAssignTypeInstr(Instruction *I, IRBuilder<> &B); void processGlobalValue(GlobalVariable &GV, IRBuilder<> &B); void processParamTypes(Function *F, IRBuilder<> &B); + Type *deduceFunParamType(Function *F, unsigned OpIdx); + Type *deduceFunParamType(Function *F, unsigned OpIdx, + std::unordered_set &FVisited); public: static char ID; @@ -169,6 +172,10 @@ static inline void reportFatalOnTokenType(const Instruction *I) { static Type *deduceElementTypeHelper(Value *I, std::unordered_set &Visited, DenseMap &DeducedElTys) { + // allow to pass nullptr as an argument + if (!I) + return nullptr; + // maybe already known auto It = DeducedElTys.find(I); if (It != DeducedElTys.end()) @@ -182,15 +189,20 @@ static Type *deduceElementTypeHelper(Value *I, // fallback value in case when we fail to deduce a type Type *Ty = nullptr; // look for known basic patterns of type inference - if (auto *Ref = dyn_cast(I)) + if (auto *Ref = dyn_cast(I)) { Ty = Ref->getAllocatedType(); - else if (auto *Ref = dyn_cast(I)) + } else if (auto *Ref = dyn_cast(I)) { Ty = Ref->getResultElementType(); - else if (auto *Ref = dyn_cast(I)) + } else if (auto *Ref = dyn_cast(I)) { Ty = Ref->getValueType(); - else if (auto *Ref = dyn_cast(I)) + } else if (auto *Ref = dyn_cast(I)) { Ty = deduceElementTypeHelper(Ref->getPointerOperand(), Visited, DeducedElTys); + } else if (auto *Ref = dyn_cast(I)) { + if (Type *Src = Ref->getSrcTy(), *Dest = Ref->getDestTy(); + isPointerTy(Src) && isPointerTy(Dest)) + Ty = deduceElementTypeHelper(Ref->getOperand(0), Visited, DeducedElTys); + } // remember the found relationship if (Ty) @@ -795,61 +807,80 @@ void SPIRVEmitIntrinsics::processInstrAfterVisit(Instruction *I, } } -void SPIRVEmitIntrinsics::processParamTypes(Function *F, IRBuilder<> &B) { - DenseMap Args; - unsigned i = 0; - for (Argument &Arg : F->args()) { - if (isUntypedPointerTy(Arg.getType()) && - DeducedElTys.find(&Arg) == DeducedElTys.end() && - !HasPointeeTypeAttr(&Arg)) - Args[i] = &Arg; - i++; - } - if (Args.size() == 0) - return; +Type *SPIRVEmitIntrinsics::deduceFunParamType(Function *F, unsigned OpIdx) { + std::unordered_set FVisited; + return deduceFunParamType(F, OpIdx, FVisited); +} + +Type *SPIRVEmitIntrinsics::deduceFunParamType( + Function *F, unsigned OpIdx, std::unordered_set &FVisited) { + // maybe a cycle + if (FVisited.find(F) != FVisited.end()) + return nullptr; + FVisited.insert(F); - // Args contains opaque pointers without element type definition - B.SetInsertPointPastAllocas(F); std::unordered_set Visited; + SmallVector> Lookup; + // search in function's call sites for (User *U : F->users()) { CallInst *CI = dyn_cast(U); - if (!CI) + if (!CI || OpIdx >= CI->arg_size()) continue; - for (unsigned OpIdx = 0; OpIdx < CI->arg_size() && Args.size() > 0; - OpIdx++) { - auto It = Args.find(OpIdx); - Argument *Arg = It == Args.end() ? nullptr : It->second; - if (!Arg) - continue; - Value *OpArg = CI->getArgOperand(OpIdx); - if (!isPointerTy(OpArg->getType())) + Value *OpArg = CI->getArgOperand(OpIdx); + if (!isPointerTy(OpArg->getType())) + continue; + // maybe we already know operand's element type + if (auto It = DeducedElTys.find(OpArg); It != DeducedElTys.end()) + return It->second; + // search in actual parameter's users + for (User *OpU : OpArg->users()) { + Instruction *Inst = dyn_cast(OpU); + if (!Inst || Inst == CI) continue; - // maybe we already know the operand's element type - auto DeducedIt = DeducedElTys.find(OpArg); - Type *ElemTy = - DeducedIt == DeducedElTys.end() ? nullptr : DeducedIt->second; - if (!ElemTy) { - for (User *OpU : OpArg->users()) { - if (Instruction *Inst = dyn_cast(OpU)) { - Visited.clear(); - ElemTy = deduceElementTypeHelper(Inst, Visited, DeducedElTys); - if (ElemTy) - break; - } - } + Visited.clear(); + if (Type *Ty = deduceElementTypeHelper(Inst, Visited, DeducedElTys)) + return Ty; + } + // check if it's a formal parameter of the outer function + if (!CI->getParent() || !CI->getParent()->getParent()) + continue; + Function *OuterF = CI->getParent()->getParent(); + if (FVisited.find(OuterF) != FVisited.end()) + continue; + for (unsigned i = 0; i < OuterF->arg_size(); ++i) { + if (OuterF->getArg(i) == OpArg) { + Lookup.push_back(std::make_pair(OuterF, i)); + break; } - if (ElemTy) { - unsigned AddressSpace = getPointerAddressSpace(Arg->getType()); + } + } + + // search in function parameters + for (auto &Pair : Lookup) { + if (Type *Ty = deduceFunParamType(Pair.first, Pair.second, FVisited)) + return Ty; + } + + return nullptr; +} + +void SPIRVEmitIntrinsics::processParamTypes(Function *F, IRBuilder<> &B) { + B.SetInsertPointPastAllocas(F); + DenseMap Args; + for (unsigned OpIdx = 0; OpIdx < F->arg_size(); ++OpIdx) { + Argument *Arg = F->getArg(OpIdx); + if (isUntypedPointerTy(Arg->getType()) && + DeducedElTys.find(Arg) == DeducedElTys.end() && + !HasPointeeTypeAttr(Arg)) { + if (Type *ElemTy = deduceFunParamType(F, OpIdx)) { CallInst *AssignPtrTyCI = buildIntrWithMD( Intrinsic::spv_assign_ptr_type, {Arg->getType()}, - Constant::getNullValue(ElemTy), Arg, {B.getInt32(AddressSpace)}, B); + Constant::getNullValue(ElemTy), Arg, + {B.getInt32(getPointerAddressSpace(Arg->getType()))}, B); DeducedElTys[AssignPtrTyCI] = ElemTy; DeducedElTys[Arg] = ElemTy; - Args.erase(It); } } - if (Args.size() == 0) - break; } } diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp index 42f8397a3023..ee52163a5d12 100644 --- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp @@ -479,6 +479,7 @@ Register SPIRVGlobalRegistry::buildGlobalVariable( GVar = M->getGlobalVariable(Name); if (GVar == nullptr) { const Type *Ty = getTypeForSPIRVType(BaseType); // TODO: check type. + // Module takes ownership of the global var. GVar = new GlobalVariable(*M, const_cast(Ty), false, GlobalValue::ExternalLinkage, nullptr, Twine(Name)); diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp index 5bb8f6084f96..39228e2196b3 100644 --- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp @@ -499,6 +499,7 @@ bool SPIRVInstructionSelector::spvSelect(Register ResVReg, assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); Register GV = I.getOperand(1).getReg(); MachineRegisterInfo::def_instr_iterator II = MRI->def_instr_begin(GV); + (void)II; assert(((*II).getOpcode() == TargetOpcode::G_GLOBAL_VALUE || (*II).getOpcode() == TargetOpcode::COPY || (*II).getOpcode() == SPIRV::OpVariable) && @@ -771,10 +772,13 @@ bool SPIRVInstructionSelector::selectMemOperation(Register ResVReg, SPIRVType *VarTy = GR.getOrCreateSPIRVPointerType( ArrTy, I, TII, SPIRV::StorageClass::UniformConstant); // TODO: check if we have such GV, add init, use buildGlobalVariable. - Type *LLVMArrTy = ArrayType::get( - IntegerType::get(GR.CurMF->getFunction().getContext(), 8), Num); - GlobalVariable *GV = - new GlobalVariable(LLVMArrTy, true, GlobalValue::InternalLinkage); + Function &CurFunction = GR.CurMF->getFunction(); + Type *LLVMArrTy = + ArrayType::get(IntegerType::get(CurFunction.getContext(), 8), Num); + // Module takes ownership of the global var. + GlobalVariable *GV = new GlobalVariable(*CurFunction.getParent(), LLVMArrTy, + true, GlobalValue::InternalLinkage, + Constant::getNullValue(LLVMArrTy)); Register VarReg = MRI->createGenericVirtualRegister(LLT::scalar(32)); GR.add(GV, GR.CurMF, VarReg); diff --git a/llvm/test/CodeGen/SPIRV/pointers/type-deduce-by-call-chain.ll b/llvm/test/CodeGen/SPIRV/pointers/type-deduce-by-call-chain.ll new file mode 100644 index 000000000000..703f1e22a032 --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/pointers/type-deduce-by-call-chain.ll @@ -0,0 +1,52 @@ +; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} + +; CHECK-SPIRV-DAG: OpName %[[ArgCum:.*]] "_arg_cum" +; CHECK-SPIRV-DAG: OpName %[[FunTest:.*]] "test" +; CHECK-SPIRV-DAG: OpName %[[Addr:.*]] "addr" +; CHECK-SPIRV-DAG: OpName %[[StubObj:.*]] "stub_object" +; CHECK-SPIRV-DAG: OpName %[[MemOrder:.*]] "mem_order" +; CHECK-SPIRV-DAG: OpName %[[FooStub:.*]] "foo_stub" +; CHECK-SPIRV-DAG: OpName %[[FooObj:.*]] "foo_object" +; CHECK-SPIRV-DAG: OpName %[[FooMemOrder:.*]] "mem_order" +; CHECK-SPIRV-DAG: OpName %[[FooFunc:.*]] "foo" +; CHECK-SPIRV-DAG: %[[TyLong:.*]] = OpTypeInt 32 0 +; CHECK-SPIRV-DAG: %[[TyVoid:.*]] = OpTypeVoid +; CHECK-SPIRV-DAG: %[[TyPtrLong:.*]] = OpTypePointer CrossWorkgroup %[[TyLong]] +; CHECK-SPIRV-DAG: %[[TyFunPtrLong:.*]] = OpTypeFunction %[[TyVoid]] %[[TyPtrLong]] +; CHECK-SPIRV-DAG: %[[TyGenPtrLong:.*]] = OpTypePointer Generic %[[TyLong]] +; CHECK-SPIRV-DAG: %[[TyFunGenPtrLongLong:.*]] = OpTypeFunction %[[TyVoid]] %[[TyGenPtrLong]] %[[TyLong]] +; CHECK-SPIRV-DAG: %[[Const3:.*]] = OpConstant %[[TyLong]] 3 +; CHECK-SPIRV: %[[FunTest]] = OpFunction %[[TyVoid]] None %[[TyFunPtrLong]] +; CHECK-SPIRV: %[[ArgCum]] = OpFunctionParameter %[[TyPtrLong]] +; CHECK-SPIRV: OpFunctionCall %[[TyVoid]] %[[FooFunc]] %[[Addr]] %[[Const3]] +; CHECK-SPIRV: %[[FooStub]] = OpFunction %[[TyVoid]] None %[[TyFunGenPtrLongLong]] +; CHECK-SPIRV: %[[StubObj]] = OpFunctionParameter %[[TyGenPtrLong]] +; CHECK-SPIRV: %[[MemOrder]] = OpFunctionParameter %[[TyLong]] +; CHECK-SPIRV: %[[FooFunc]] = OpFunction %[[TyVoid]] None %[[TyFunGenPtrLongLong]] +; CHECK-SPIRV: %[[FooObj]] = OpFunctionParameter %[[TyGenPtrLong]] +; CHECK-SPIRV: %[[FooMemOrder]] = OpFunctionParameter %[[TyLong]] +; CHECK-SPIRV: OpFunctionCall %[[TyVoid]] %[[FooStub]] %[[FooObj]] %[[FooMemOrder]] + +define spir_kernel void @test(ptr addrspace(1) noundef align 4 %_arg_cum) { +entry: + %lptr = getelementptr inbounds i32, ptr addrspace(1) %_arg_cum, i64 1 + %addr = addrspacecast ptr addrspace(1) %lptr to ptr addrspace(4) + %object = bitcast ptr addrspace(4) %addr to ptr addrspace(4) + call spir_func void @foo(ptr addrspace(4) %object, i32 3) + ret void +} + +define void @foo_stub(ptr addrspace(4) noundef %stub_object, i32 noundef %mem_order) { +entry: + %object.addr = alloca ptr addrspace(4) + %object.addr.ascast = addrspacecast ptr %object.addr to ptr addrspace(4) + store ptr addrspace(4) %stub_object, ptr addrspace(4) %object.addr.ascast + ret void +} + +define void @foo(ptr addrspace(4) noundef %foo_object, i32 noundef %mem_order) { + tail call void @foo_stub(ptr addrspace(4) noundef %foo_object, i32 noundef %mem_order) + ret void +} + -- GitLab From b0d03ccc0855f2bff39160f25fcde06aae07cace Mon Sep 17 00:00:00 2001 From: Vyacheslav Levytskyy Date: Mon, 25 Mar 2024 10:14:46 +0100 Subject: [PATCH 192/306] [SPIR-V] Fix illegal OpConstantComposite instruction with non-const constituents in SPIR-V Backend (#86352) This PR fixes illegal use of OpConstantComposite with non-constant constituents. The test attached to the PR is able now to satisfy `spirv-val` check. Before the fix SPIR-V Backend produced for the attached test case a pattern like ``` %a = OpVariable %_ptr_CrossWorkgroup_uint CrossWorkgroup %uint_123 %11 = OpConstantComposite %_struct_6 %a %a ``` so that `spirv-val` complained with ``` error: line 25: OpConstantComposite Constituent '10[%a]' is not a constant or undef. %11 = OpConstantComposite %_struct_6 %a %a ``` --- .../Target/SPIRV/SPIRVDuplicatesTracker.cpp | 1 + .../lib/Target/SPIRV/SPIRVDuplicatesTracker.h | 9 ++ llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp | 1 + llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h | 8 ++ .../Target/SPIRV/SPIRVInstructionSelector.cpp | 90 +++++++++++++++---- llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp | 1 + .../lib/Target/SPIRV/SPIRVSymbolicOperands.td | 1 + .../SPIRV/pointers/struct-opaque-pointers.ll | 2 +- 8 files changed, 96 insertions(+), 17 deletions(-) diff --git a/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.cpp b/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.cpp index d82fb2df4539..7c32bb1968ef 100644 --- a/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.cpp @@ -39,6 +39,7 @@ void SPIRVGeneralDuplicatesTracker::buildDepsGraph( prebuildReg2Entry(GT, Reg2Entry); prebuildReg2Entry(FT, Reg2Entry); prebuildReg2Entry(AT, Reg2Entry); + prebuildReg2Entry(MT, Reg2Entry); prebuildReg2Entry(ST, Reg2Entry); for (auto &Op2E : Reg2Entry) { diff --git a/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.h b/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.h index 96cc621791e9..2ec3fb35ca04 100644 --- a/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.h +++ b/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.h @@ -262,6 +262,7 @@ class SPIRVGeneralDuplicatesTracker { SPIRVDuplicatesTracker GT; SPIRVDuplicatesTracker FT; SPIRVDuplicatesTracker AT; + SPIRVDuplicatesTracker MT; SPIRVDuplicatesTracker ST; // NOTE: using MOs instead of regs to get rid of MF dependency to be able @@ -306,6 +307,10 @@ public: AT.add(Arg, MF, R); } + void add(const MachineInstr *MI, const MachineFunction *MF, Register R) { + MT.add(MI, MF, R); + } + void add(const SPIRV::SpecialTypeDescriptor &TD, const MachineFunction *MF, Register R) { ST.add(TD, MF, R); @@ -337,6 +342,10 @@ public: return AT.find(const_cast(Arg), MF); } + Register find(const MachineInstr *MI, const MachineFunction *MF) { + return MT.find(const_cast(MI), MF); + } + Register find(const SPIRV::SpecialTypeDescriptor &TD, const MachineFunction *MF) { return ST.find(TD, MF); diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp index ee52163a5d12..db66ed4f0e0f 100644 --- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp @@ -123,6 +123,7 @@ SPIRVType *SPIRVGlobalRegistry::getOpTypeVector(uint32_t NumElems, SPIRVType *ElemType, MachineIRBuilder &MIRBuilder) { auto EleOpc = ElemType->getOpcode(); + (void)EleOpc; assert((EleOpc == SPIRV::OpTypeInt || EleOpc == SPIRV::OpTypeFloat || EleOpc == SPIRV::OpTypeBool) && "Invalid vector element type"); diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h index da480b22a525..ed0f90ff89ce 100644 --- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h +++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h @@ -94,6 +94,14 @@ public: DT.add(Arg, MF, R); } + void add(const MachineInstr *MI, MachineFunction *MF, Register R) { + DT.add(MI, MF, R); + } + + Register find(const MachineInstr *MI, MachineFunction *MF) { + return DT.find(MI, MF); + } + Register find(const Constant *C, MachineFunction *MF) { return DT.find(C, MF); } diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp index 39228e2196b3..505b19a4d66e 100644 --- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp @@ -231,6 +231,9 @@ private: Register buildZerosVal(const SPIRVType *ResType, MachineInstr &I) const; Register buildOnesVal(bool AllOnes, const SPIRVType *ResType, MachineInstr &I) const; + + bool wrapIntoSpecConstantOp(MachineInstr &I, + SmallVector &CompositeArgs) const; }; } // end anonymous namespace @@ -1249,6 +1252,24 @@ static unsigned getArrayComponentCount(MachineRegisterInfo *MRI, return N; } +// Return true if the type represents a constant register +static bool isConstReg(MachineRegisterInfo *MRI, SPIRVType *OpDef) { + if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && + OpDef->getOperand(1).isReg()) { + if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) + OpDef = RefDef; + } + return OpDef->getOpcode() == TargetOpcode::G_CONSTANT || + OpDef->getOpcode() == TargetOpcode::G_FCONSTANT; +} + +// Return true if the virtual register represents a constant +static bool isConstReg(MachineRegisterInfo *MRI, Register OpReg) { + if (SPIRVType *OpDef = MRI->getVRegDef(OpReg)) + return isConstReg(MRI, OpDef); + return false; +} + bool SPIRVInstructionSelector::selectSplatVector(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { @@ -1266,16 +1287,7 @@ bool SPIRVInstructionSelector::selectSplatVector(Register ResVReg, // check if we may construct a constant vector Register OpReg = I.getOperand(OpIdx).getReg(); - bool IsConst = false; - if (SPIRVType *OpDef = MRI->getVRegDef(OpReg)) { - if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && - OpDef->getOperand(1).isReg()) { - if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) - OpDef = RefDef; - } - IsConst = OpDef->getOpcode() == TargetOpcode::G_CONSTANT || - OpDef->getOpcode() == TargetOpcode::G_FCONSTANT; - } + bool IsConst = isConstReg(MRI, OpReg); if (!IsConst && N < 2) report_fatal_error( @@ -1628,6 +1640,48 @@ bool SPIRVInstructionSelector::selectGEP(Register ResVReg, return Res.constrainAllUses(TII, TRI, RBI); } +// Maybe wrap a value into OpSpecConstantOp +bool SPIRVInstructionSelector::wrapIntoSpecConstantOp( + MachineInstr &I, SmallVector &CompositeArgs) const { + bool Result = true; + unsigned Lim = I.getNumExplicitOperands(); + for (unsigned i = I.getNumExplicitDefs() + 1; i < Lim; ++i) { + Register OpReg = I.getOperand(i).getReg(); + SPIRVType *OpDefine = MRI->getVRegDef(OpReg); + SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpReg); + if (!OpDefine || !OpType || isConstReg(MRI, OpDefine) || + OpDefine->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST) { + // The case of G_ADDRSPACE_CAST inside spv_const_composite() is processed + // by selectAddrSpaceCast() + CompositeArgs.push_back(OpReg); + continue; + } + MachineFunction *MF = I.getMF(); + Register WrapReg = GR.find(OpDefine, MF); + if (WrapReg.isValid()) { + CompositeArgs.push_back(WrapReg); + continue; + } + // Create a new register for the wrapper + WrapReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); + GR.add(OpDefine, MF, WrapReg); + CompositeArgs.push_back(WrapReg); + // Decorate the wrapper register and generate a new instruction + MRI->setType(WrapReg, LLT::pointer(0, 32)); + GR.assignSPIRVTypeToVReg(OpType, WrapReg, *MF); + MachineBasicBlock &BB = *I.getParent(); + Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) + .addDef(WrapReg) + .addUse(GR.getSPIRVTypeID(OpType)) + .addImm(static_cast(SPIRV::Opcode::Bitcast)) + .addUse(OpReg) + .constrainAllUses(TII, TRI, RBI); + if (!Result) + break; + } + return Result; +} + bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { @@ -1666,17 +1720,21 @@ bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg, case Intrinsic::spv_const_composite: { // If no values are attached, the composite is null constant. bool IsNull = I.getNumExplicitDefs() + 1 == I.getNumExplicitOperands(); - unsigned Opcode = - IsNull ? SPIRV::OpConstantNull : SPIRV::OpConstantComposite; + // Select a proper instruction. + unsigned Opcode = SPIRV::OpConstantNull; + SmallVector CompositeArgs; + if (!IsNull) { + Opcode = SPIRV::OpConstantComposite; + if (!wrapIntoSpecConstantOp(I, CompositeArgs)) + return false; + } auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) .addDef(ResVReg) .addUse(GR.getSPIRVTypeID(ResType)); // skip type MD node we already used when generated assign.type for this if (!IsNull) { - for (unsigned i = I.getNumExplicitDefs() + 1; - i < I.getNumExplicitOperands(); ++i) { - MIB.addUse(I.getOperand(i).getReg()); - } + for (Register OpReg : CompositeArgs) + MIB.addUse(OpReg); } return MIB.constrainAllUses(TII, TRI, RBI); } diff --git a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp index d547f91ba4a5..1f0d8d8cd43a 100644 --- a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp @@ -543,6 +543,7 @@ static void processSwitches(MachineFunction &MF, SPIRVGlobalRegistry *GR, Register Dst = ICMP->getOperand(0).getReg(); MachineOperand &PredOp = ICMP->getOperand(1); const auto CC = static_cast(PredOp.getPredicate()); + (void)CC; assert((CC == CmpInst::ICMP_EQ || CC == CmpInst::ICMP_ULE) && MRI.hasOneUse(Dst) && MRI.hasOneDef(CompareReg)); uint64_t Value = getIConstVal(ICMP->getOperand(3).getReg(), &MRI); diff --git a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td index 8dbbd9049844..ff102e318469 100644 --- a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td +++ b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td @@ -1611,3 +1611,4 @@ multiclass OpcodeOperand value> { // TODO: implement other mnemonics. defm InBoundsPtrAccessChain : OpcodeOperand<70>; defm PtrCastToGeneric : OpcodeOperand<121>; +defm Bitcast : OpcodeOperand<124>; diff --git a/llvm/test/CodeGen/SPIRV/pointers/struct-opaque-pointers.ll b/llvm/test/CodeGen/SPIRV/pointers/struct-opaque-pointers.ll index d426fc4dfd4e..ce3ab8895a59 100644 --- a/llvm/test/CodeGen/SPIRV/pointers/struct-opaque-pointers.ll +++ b/llvm/test/CodeGen/SPIRV/pointers/struct-opaque-pointers.ll @@ -1,5 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s -; TODO: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; CHECK: %[[TyInt8:.*]] = OpTypeInt 8 0 ; CHECK: %[[TyInt8Ptr:.*]] = OpTypePointer {{[a-zA-Z]+}} %[[TyInt8]] -- GitLab From cbbbab349e9c412729c3969008cdcb677cc55790 Mon Sep 17 00:00:00 2001 From: Orlando Cazalet-Hyams Date: Fri, 15 Mar 2024 12:41:32 +0000 Subject: [PATCH 193/306] [RemoveDIs] Enable direct-to-bitcode writing by default Follow on from #83251. This patch simply enables the behaviour by default in order to provide an easily revertible capstone. --- llvm/lib/IR/BasicBlock.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/IR/BasicBlock.cpp b/llvm/lib/IR/BasicBlock.cpp index f088c7a2cc4e..ae99267f5ba8 100644 --- a/llvm/lib/IR/BasicBlock.cpp +++ b/llvm/lib/IR/BasicBlock.cpp @@ -39,7 +39,7 @@ cl::opt bool WriteNewDbgInfoFormatToBitcode /*set default value in cl::init() below*/; cl::opt WriteNewDbgInfoFormatToBitcode2( "write-experimental-debuginfo-iterators-to-bitcode", cl::Hidden, - cl::location(WriteNewDbgInfoFormatToBitcode), cl::init(false)); + cl::location(WriteNewDbgInfoFormatToBitcode), cl::init(true)); DbgMarker *BasicBlock::createMarker(Instruction *I) { assert(IsNewDbgInfoFormat && -- GitLab From 3cb024198f6a028089779c0a3cb4d8e753e87c73 Mon Sep 17 00:00:00 2001 From: Shih-Po Hung Date: Mon, 25 Mar 2024 17:17:36 +0800 Subject: [PATCH 194/306] [RISCV][CostModel] Estimate cost of llvm.vector.reduce.fmaximum/fminimum (#80697) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ‘llvm.vector.reduce.fmaximum/fminimum.*’ intrinsics propagate NaNs if any element of the vector is a NaN. Following #79402, the patch adds the cost for NaN check (vmfne + vcpop) --- .../Target/RISCV/RISCVTargetTransformInfo.cpp | 45 +++++++++ .../CostModel/RISCV/reduce-fmaximum.ll | 91 +++++++++++++------ .../CostModel/RISCV/reduce-fminimum.ll | 52 +++++------ 3 files changed, 136 insertions(+), 52 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp index 8f46fdc2f7ca..f75b3d3caa62 100644 --- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp @@ -1001,6 +1001,51 @@ RISCVTTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty, return getArithmeticReductionCost(Instruction::And, Ty, FMF, CostKind); } + if (IID == Intrinsic::maximum || IID == Intrinsic::minimum) { + SmallVector Opcodes; + InstructionCost ExtraCost = 0; + switch (IID) { + case Intrinsic::maximum: + if (FMF.noNaNs()) { + Opcodes = {RISCV::VFREDMAX_VS, RISCV::VFMV_F_S}; + } else { + Opcodes = {RISCV::VMFNE_VV, RISCV::VCPOP_M, RISCV::VFREDMAX_VS, + RISCV::VFMV_F_S}; + // Cost of Canonical Nan + branch + // lui a0, 523264 + // fmv.w.x fa0, a0 + Type *DstTy = Ty->getScalarType(); + const unsigned EltTyBits = DstTy->getScalarSizeInBits(); + Type *SrcTy = IntegerType::getIntNTy(DstTy->getContext(), EltTyBits); + ExtraCost = 1 + + getCastInstrCost(Instruction::UIToFP, DstTy, SrcTy, + TTI::CastContextHint::None, CostKind) + + getCFInstrCost(Instruction::Br, CostKind); + } + break; + + case Intrinsic::minimum: + if (FMF.noNaNs()) { + Opcodes = {RISCV::VFREDMIN_VS, RISCV::VFMV_F_S}; + } else { + Opcodes = {RISCV::VMFNE_VV, RISCV::VCPOP_M, RISCV::VFREDMIN_VS, + RISCV::VFMV_F_S}; + // Cost of Canonical Nan + branch + // lui a0, 523264 + // fmv.w.x fa0, a0 + Type *DstTy = Ty->getScalarType(); + const unsigned EltTyBits = DL.getTypeSizeInBits(DstTy); + Type *SrcTy = IntegerType::getIntNTy(DstTy->getContext(), EltTyBits); + ExtraCost = 1 + + getCastInstrCost(Instruction::UIToFP, DstTy, SrcTy, + TTI::CastContextHint::None, CostKind) + + getCFInstrCost(Instruction::Br, CostKind); + } + break; + } + return ExtraCost + getRISCVInstructionCost(Opcodes, LT.second, CostKind); + } + // IR Reduction is composed by two vmv and one rvv reduction instruction. InstructionCost BaseCost = 2; diff --git a/llvm/test/Analysis/CostModel/RISCV/reduce-fmaximum.ll b/llvm/test/Analysis/CostModel/RISCV/reduce-fmaximum.ll index 1618c3833a97..f91f13b2d9ec 100644 --- a/llvm/test/Analysis/CostModel/RISCV/reduce-fmaximum.ll +++ b/llvm/test/Analysis/CostModel/RISCV/reduce-fmaximum.ll @@ -6,23 +6,37 @@ define float @reduce_fmaximum_f32(float %arg) { ; CHECK-LABEL: 'reduce_fmaximum_f32' -; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V2 = call float @llvm.vector.reduce.fmaximum.v2f32(<2 x float> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V4 = call float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V8 = call float @llvm.vector.reduce.fmaximum.v8f32(<8 x float> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16 = call float @llvm.vector.reduce.fmaximum.v16f32(<16 x float> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32 = call float @llvm.vector.reduce.fmaximum.v32f32(<32 x float> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V64 = call float @llvm.vector.reduce.fmaximum.v64f32(<64 x float> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V128 = call float @llvm.vector.reduce.fmaximum.v128f32(<128 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V2 = call float @llvm.vector.reduce.fmaximum.v2f32(<2 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V4 = call float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V8 = call float @llvm.vector.reduce.fmaximum.v8f32(<8 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V16 = call float @llvm.vector.reduce.fmaximum.v16f32(<16 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %V32 = call float @llvm.vector.reduce.fmaximum.v32f32(<32 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V64 = call float @llvm.vector.reduce.fmaximum.v64f32(<64 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V128 = call float @llvm.vector.reduce.fmaximum.v128f32(<128 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %1 = call fast float @llvm.vector.reduce.fmaximum.v2f32(<2 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %2 = call fast float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %3 = call fast float @llvm.vector.reduce.fmaximum.v8f32(<8 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %4 = call fast float @llvm.vector.reduce.fmaximum.v16f32(<16 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %5 = call fast float @llvm.vector.reduce.fmaximum.v32f32(<32 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %6 = call fast float @llvm.vector.reduce.fmaximum.v64f32(<64 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %7 = call fast float @llvm.vector.reduce.fmaximum.v128f32(<128 x float> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret float undef ; ; SIZE-LABEL: 'reduce_fmaximum_f32' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V2 = call float @llvm.vector.reduce.fmaximum.v2f32(<2 x float> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V4 = call float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V8 = call float @llvm.vector.reduce.fmaximum.v8f32(<8 x float> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V16 = call float @llvm.vector.reduce.fmaximum.v16f32(<16 x float> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V32 = call float @llvm.vector.reduce.fmaximum.v32f32(<32 x float> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V64 = call float @llvm.vector.reduce.fmaximum.v64f32(<64 x float> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V128 = call float @llvm.vector.reduce.fmaximum.v128f32(<128 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V2 = call float @llvm.vector.reduce.fmaximum.v2f32(<2 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V4 = call float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V8 = call float @llvm.vector.reduce.fmaximum.v8f32(<8 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V16 = call float @llvm.vector.reduce.fmaximum.v16f32(<16 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32 = call float @llvm.vector.reduce.fmaximum.v32f32(<32 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V64 = call float @llvm.vector.reduce.fmaximum.v64f32(<64 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V128 = call float @llvm.vector.reduce.fmaximum.v128f32(<128 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %1 = call fast float @llvm.vector.reduce.fmaximum.v2f32(<2 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %2 = call fast float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %3 = call fast float @llvm.vector.reduce.fmaximum.v8f32(<8 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %4 = call fast float @llvm.vector.reduce.fmaximum.v16f32(<16 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %5 = call fast float @llvm.vector.reduce.fmaximum.v32f32(<32 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %6 = call fast float @llvm.vector.reduce.fmaximum.v64f32(<64 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %7 = call fast float @llvm.vector.reduce.fmaximum.v128f32(<128 x float> undef) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret float undef ; %V2 = call float @llvm.vector.reduce.fmaximum.v2f32(<2 x float> undef) @@ -32,6 +46,13 @@ define float @reduce_fmaximum_f32(float %arg) { %V32 = call float @llvm.vector.reduce.fmaximum.v32f32(<32 x float> undef) %V64 = call float @llvm.vector.reduce.fmaximum.v64f32(<64 x float> undef) %V128 = call float @llvm.vector.reduce.fmaximum.v128f32(<128 x float> undef) +call fast float @llvm.vector.reduce.fmaximum.v2f32(<2 x float> undef) +call fast float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> undef) +call fast float @llvm.vector.reduce.fmaximum.v8f32(<8 x float> undef) +call fast float @llvm.vector.reduce.fmaximum.v16f32(<16 x float> undef) +call fast float @llvm.vector.reduce.fmaximum.v32f32(<32 x float> undef) +call fast float @llvm.vector.reduce.fmaximum.v64f32(<64 x float> undef) +call fast float @llvm.vector.reduce.fmaximum.v128f32(<128 x float> undef) ret float undef } declare float @llvm.vector.reduce.fmaximum.v2f32(<2 x float>) @@ -44,21 +65,33 @@ declare float @llvm.vector.reduce.fmaximum.v128f32(<128 x float>) define double @reduce_fmaximum_f64(double %arg) { ; CHECK-LABEL: 'reduce_fmaximum_f64' -; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V2 = call double @llvm.vector.reduce.fmaximum.v2f64(<2 x double> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V4 = call double @llvm.vector.reduce.fmaximum.v4f64(<4 x double> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V8 = call double @llvm.vector.reduce.fmaximum.v8f64(<8 x double> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16 = call double @llvm.vector.reduce.fmaximum.v16f64(<16 x double> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32 = call double @llvm.vector.reduce.fmaximum.v32f64(<32 x double> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V64 = call double @llvm.vector.reduce.fmaximum.v64f64(<64 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V2 = call double @llvm.vector.reduce.fmaximum.v2f64(<2 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V4 = call double @llvm.vector.reduce.fmaximum.v4f64(<4 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V8 = call double @llvm.vector.reduce.fmaximum.v8f64(<8 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V16 = call double @llvm.vector.reduce.fmaximum.v16f64(<16 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %V32 = call double @llvm.vector.reduce.fmaximum.v32f64(<32 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %V64 = call double @llvm.vector.reduce.fmaximum.v64f64(<64 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %1 = call fast double @llvm.vector.reduce.fmaximum.v2f64(<2 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %2 = call fast double @llvm.vector.reduce.fmaximum.v4f64(<4 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %3 = call fast double @llvm.vector.reduce.fmaximum.v8f64(<8 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %4 = call fast double @llvm.vector.reduce.fmaximum.v16f64(<16 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %5 = call fast double @llvm.vector.reduce.fmaximum.v32f64(<32 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %6 = call fast double @llvm.vector.reduce.fmaximum.v64f64(<64 x double> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret double undef ; ; SIZE-LABEL: 'reduce_fmaximum_f64' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V2 = call double @llvm.vector.reduce.fmaximum.v2f64(<2 x double> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V4 = call double @llvm.vector.reduce.fmaximum.v4f64(<4 x double> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V8 = call double @llvm.vector.reduce.fmaximum.v8f64(<8 x double> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V16 = call double @llvm.vector.reduce.fmaximum.v16f64(<16 x double> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V32 = call double @llvm.vector.reduce.fmaximum.v32f64(<32 x double> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V64 = call double @llvm.vector.reduce.fmaximum.v64f64(<64 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V2 = call double @llvm.vector.reduce.fmaximum.v2f64(<2 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V4 = call double @llvm.vector.reduce.fmaximum.v4f64(<4 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V8 = call double @llvm.vector.reduce.fmaximum.v8f64(<8 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V16 = call double @llvm.vector.reduce.fmaximum.v16f64(<16 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32 = call double @llvm.vector.reduce.fmaximum.v32f64(<32 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V64 = call double @llvm.vector.reduce.fmaximum.v64f64(<64 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %1 = call fast double @llvm.vector.reduce.fmaximum.v2f64(<2 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %2 = call fast double @llvm.vector.reduce.fmaximum.v4f64(<4 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %3 = call fast double @llvm.vector.reduce.fmaximum.v8f64(<8 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %4 = call fast double @llvm.vector.reduce.fmaximum.v16f64(<16 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %5 = call fast double @llvm.vector.reduce.fmaximum.v32f64(<32 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %6 = call fast double @llvm.vector.reduce.fmaximum.v64f64(<64 x double> undef) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret double undef ; %V2 = call double @llvm.vector.reduce.fmaximum.v2f64(<2 x double> undef) @@ -67,6 +100,12 @@ define double @reduce_fmaximum_f64(double %arg) { %V16 = call double @llvm.vector.reduce.fmaximum.v16f64(<16 x double> undef) %V32 = call double @llvm.vector.reduce.fmaximum.v32f64(<32 x double> undef) %V64 = call double @llvm.vector.reduce.fmaximum.v64f64(<64 x double> undef) +call fast double @llvm.vector.reduce.fmaximum.v2f64(<2 x double> undef) +call fast double @llvm.vector.reduce.fmaximum.v4f64(<4 x double> undef) +call fast double @llvm.vector.reduce.fmaximum.v8f64(<8 x double> undef) +call fast double @llvm.vector.reduce.fmaximum.v16f64(<16 x double> undef) +call fast double @llvm.vector.reduce.fmaximum.v32f64(<32 x double> undef) +call fast double @llvm.vector.reduce.fmaximum.v64f64(<64 x double> undef) ret double undef } declare double @llvm.vector.reduce.fmaximum.v2f64(<2 x double>) diff --git a/llvm/test/Analysis/CostModel/RISCV/reduce-fminimum.ll b/llvm/test/Analysis/CostModel/RISCV/reduce-fminimum.ll index 35b18645b1f2..86b84025ad54 100644 --- a/llvm/test/Analysis/CostModel/RISCV/reduce-fminimum.ll +++ b/llvm/test/Analysis/CostModel/RISCV/reduce-fminimum.ll @@ -6,23 +6,23 @@ define float @reduce_fmaximum_f32(float %arg) { ; CHECK-LABEL: 'reduce_fmaximum_f32' -; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V2 = call float @llvm.vector.reduce.fminimum.v2f32(<2 x float> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V4 = call float @llvm.vector.reduce.fminimum.v4f32(<4 x float> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V8 = call float @llvm.vector.reduce.fminimum.v8f32(<8 x float> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16 = call float @llvm.vector.reduce.fminimum.v16f32(<16 x float> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32 = call float @llvm.vector.reduce.fminimum.v32f32(<32 x float> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V64 = call float @llvm.vector.reduce.fminimum.v64f32(<64 x float> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V128 = call float @llvm.vector.reduce.fminimum.v128f32(<128 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V2 = call float @llvm.vector.reduce.fminimum.v2f32(<2 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V4 = call float @llvm.vector.reduce.fminimum.v4f32(<4 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V8 = call float @llvm.vector.reduce.fminimum.v8f32(<8 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V16 = call float @llvm.vector.reduce.fminimum.v16f32(<16 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %V32 = call float @llvm.vector.reduce.fminimum.v32f32(<32 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V64 = call float @llvm.vector.reduce.fminimum.v64f32(<64 x float> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V128 = call float @llvm.vector.reduce.fminimum.v128f32(<128 x float> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret float undef ; ; SIZE-LABEL: 'reduce_fmaximum_f32' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V2 = call float @llvm.vector.reduce.fminimum.v2f32(<2 x float> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V4 = call float @llvm.vector.reduce.fminimum.v4f32(<4 x float> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V8 = call float @llvm.vector.reduce.fminimum.v8f32(<8 x float> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V16 = call float @llvm.vector.reduce.fminimum.v16f32(<16 x float> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V32 = call float @llvm.vector.reduce.fminimum.v32f32(<32 x float> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V64 = call float @llvm.vector.reduce.fminimum.v64f32(<64 x float> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V128 = call float @llvm.vector.reduce.fminimum.v128f32(<128 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V2 = call float @llvm.vector.reduce.fminimum.v2f32(<2 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V4 = call float @llvm.vector.reduce.fminimum.v4f32(<4 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V8 = call float @llvm.vector.reduce.fminimum.v8f32(<8 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V16 = call float @llvm.vector.reduce.fminimum.v16f32(<16 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32 = call float @llvm.vector.reduce.fminimum.v32f32(<32 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V64 = call float @llvm.vector.reduce.fminimum.v64f32(<64 x float> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V128 = call float @llvm.vector.reduce.fminimum.v128f32(<128 x float> undef) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret float undef ; %V2 = call float @llvm.vector.reduce.fminimum.v2f32(<2 x float> undef) @@ -44,21 +44,21 @@ declare float @llvm.vector.reduce.fminimum.v128f32(<128 x float>) define double @reduce_fmaximum_f64(double %arg) { ; CHECK-LABEL: 'reduce_fmaximum_f64' -; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V2 = call double @llvm.vector.reduce.fminimum.v2f64(<2 x double> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V4 = call double @llvm.vector.reduce.fminimum.v4f64(<4 x double> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V8 = call double @llvm.vector.reduce.fminimum.v8f64(<8 x double> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16 = call double @llvm.vector.reduce.fminimum.v16f64(<16 x double> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32 = call double @llvm.vector.reduce.fminimum.v32f64(<32 x double> undef) -; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V64 = call double @llvm.vector.reduce.fminimum.v64f64(<64 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V2 = call double @llvm.vector.reduce.fminimum.v2f64(<2 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V4 = call double @llvm.vector.reduce.fminimum.v4f64(<4 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V8 = call double @llvm.vector.reduce.fminimum.v8f64(<8 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V16 = call double @llvm.vector.reduce.fminimum.v16f64(<16 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %V32 = call double @llvm.vector.reduce.fminimum.v32f64(<32 x double> undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %V64 = call double @llvm.vector.reduce.fminimum.v64f64(<64 x double> undef) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret double undef ; ; SIZE-LABEL: 'reduce_fmaximum_f64' -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V2 = call double @llvm.vector.reduce.fminimum.v2f64(<2 x double> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V4 = call double @llvm.vector.reduce.fminimum.v4f64(<4 x double> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V8 = call double @llvm.vector.reduce.fminimum.v8f64(<8 x double> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V16 = call double @llvm.vector.reduce.fminimum.v16f64(<16 x double> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V32 = call double @llvm.vector.reduce.fminimum.v32f64(<32 x double> undef) -; SIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V64 = call double @llvm.vector.reduce.fminimum.v64f64(<64 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V2 = call double @llvm.vector.reduce.fminimum.v2f64(<2 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V4 = call double @llvm.vector.reduce.fminimum.v4f64(<4 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V8 = call double @llvm.vector.reduce.fminimum.v8f64(<8 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V16 = call double @llvm.vector.reduce.fminimum.v16f64(<16 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32 = call double @llvm.vector.reduce.fminimum.v32f64(<32 x double> undef) +; SIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V64 = call double @llvm.vector.reduce.fminimum.v64f64(<64 x double> undef) ; SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret double undef ; %V2 = call double @llvm.vector.reduce.fminimum.v2f64(<2 x double> undef) -- GitLab From e550022b68fc75d32e44faa59ed8f02036cd8f26 Mon Sep 17 00:00:00 2001 From: Orlando Cazalet-Hyams Date: Mon, 25 Mar 2024 09:26:27 +0000 Subject: [PATCH 195/306] [RemoveDIs] Load into new debug info format by default in llvm-dis (#86276) Directly load all bitcode into the new debug info format in llvm-dis. This means that new-mode bitcode no longer round-trips back to old-mode after parsing, and that old-mode bitcode gets auto-upgraded to new-mode debug info (which is the current in-memory default in LLVM). --- llvm/tools/llvm-dis/llvm-dis.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/llvm/tools/llvm-dis/llvm-dis.cpp b/llvm/tools/llvm-dis/llvm-dis.cpp index 8e443318dd7d..49154dc46c57 100644 --- a/llvm/tools/llvm-dis/llvm-dis.cpp +++ b/llvm/tools/llvm-dis/llvm-dis.cpp @@ -82,6 +82,8 @@ static cl::opt PrintThinLTOIndexOnly( extern cl::opt WriteNewDbgInfoFormat; +extern cl::opt LoadBitcodeIntoNewDbgInfoFormat; + namespace { static void printDebugLoc(const DebugLoc &DL, formatted_raw_ostream &OS) { @@ -169,6 +171,10 @@ int main(int argc, char **argv) { cl::HideUnrelatedOptions({&DisCategory, &getColorCategory()}); cl::ParseCommandLineOptions(argc, argv, "llvm .bc -> .ll disassembler\n"); + // Load bitcode into the new debug info format by default. + if (LoadBitcodeIntoNewDbgInfoFormat == cl::boolOrDefault::BOU_UNSET) + LoadBitcodeIntoNewDbgInfoFormat = cl::boolOrDefault::BOU_TRUE; + LLVMContext Context; Context.setDiagnosticHandler( std::make_unique(argv[0])); -- GitLab From 2ef612050844355906e4b67d892a00bbb58c41d6 Mon Sep 17 00:00:00 2001 From: Orlando Cazalet-Hyams Date: Mon, 25 Mar 2024 09:28:01 +0000 Subject: [PATCH 196/306] [RemoveDIs] Do not load into new debug info format from bitcode by default (#86268) This is NFC right now, as the global default behaviour is also "do not load into the new debug info format by default", but we want to change that soon. Additionally unconditionally convert from the new debug info format into if we've loaded into it (e.g., if the bitcode file loaded was already in the new format). The latter change is needed because verify-uselistorder doesn't yet understand DbgRecords (it doesn't know how to map them). The former change is needed because if we load from an old debug format bitcode file but load directly into the new format _and then convert back to the old mode after_, the use-lists of the debug intrinsic functions (the functions' global value uses) change. --- .../verify-uselistorder/verify-uselistorder.cpp | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/llvm/tools/verify-uselistorder/verify-uselistorder.cpp b/llvm/tools/verify-uselistorder/verify-uselistorder.cpp index d929ae09958a..8b299c394e4e 100644 --- a/llvm/tools/verify-uselistorder/verify-uselistorder.cpp +++ b/llvm/tools/verify-uselistorder/verify-uselistorder.cpp @@ -68,6 +68,8 @@ static cl::opt cl::desc("Number of times to shuffle and verify use-lists"), cl::init(1), cl::cat(Cat)); +extern cl::opt LoadBitcodeIntoNewDbgInforFormat; + namespace { struct TempFile { @@ -169,8 +171,7 @@ std::unique_ptr TempFile::readBitcode(LLVMContext &Context) const { // verify-uselistoder currently only supports old-style debug info mode. // FIXME: Update mapping code for RemoveDIs. - assert(!ModuleOr.get()->IsNewDbgInfoFormat && - "Unexpectedly in new debug info mode"); + ModuleOr.get()->setIsNewDbgInfoFormat(false); return std::move(ModuleOr.get()); } @@ -182,7 +183,7 @@ std::unique_ptr TempFile::readAssembly(LLVMContext &Context) const { Err.print("verify-uselistorder", errs()); // verify-uselistoder currently only supports old-style debug info mode. // FIXME: Update mapping code for RemoveDIs. - assert(!M->IsNewDbgInfoFormat && "Unexpectedly in new debug info mode"); + M->setIsNewDbgInfoFormat(false); return M; } @@ -544,6 +545,10 @@ int main(int argc, char **argv) { cl::ParseCommandLineOptions(argc, argv, "llvm tool to verify use-list order\n"); + // Do not load bitcode into the new debug info format by default. + if (LoadBitcodeIntoNewDbgInforFormat == cl::boolOrDefault::BOU_UNSET) + LoadBitcodeIntoNewDbgInforFormat = cl::boolOrDefault::BOU_FALSE; + LLVMContext Context; SMDiagnostic Err; @@ -551,7 +556,7 @@ int main(int argc, char **argv) { std::unique_ptr M = parseIRFile(InputFilename, Err, Context); // verify-uselistoder currently only supports old-style debug info mode. // FIXME: Update mapping code for RemoveDIs. - assert(!M->IsNewDbgInfoFormat && "Unexpectedly in new debug info mode"); + M->setIsNewDbgInfoFormat(false); if (!M.get()) { Err.print(argv[0], errs()); -- GitLab From 8263a883342d9925a4a1fd9752efc8deda5840fc Mon Sep 17 00:00:00 2001 From: Orlando Cazalet-Hyams Date: Mon, 25 Mar 2024 09:29:19 +0000 Subject: [PATCH 197/306] [RemoveDIs] Load into new debug info format by default in llvm-link (#86274) Directly load all bitcode into the new debug info format in llvm-link. This means that new-mode bitcode no longer round-trips back to old-mode after parsing, and that old-mode bitcode gets auto-upgraded to new-mode debug info (which is the current in-memory default in LLVM). --- llvm/tools/llvm-link/llvm-link.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/llvm/tools/llvm-link/llvm-link.cpp b/llvm/tools/llvm-link/llvm-link.cpp index 9e7f2c3ebac4..9049cb5e8580 100644 --- a/llvm/tools/llvm-link/llvm-link.cpp +++ b/llvm/tools/llvm-link/llvm-link.cpp @@ -136,6 +136,8 @@ static cl::opt TryUseNewDbgInfoFormat( extern cl::opt UseNewDbgInfoFormat; +extern cl::opt LoadBitcodeIntoNewDbgInfoFormat; + static ExitOnError ExitOnErr; // Read the specified bitcode file in and return it. This routine searches the @@ -480,6 +482,10 @@ int main(int argc, char **argv) { cl::HideUnrelatedOptions({&LinkCategory, &getColorCategory()}); cl::ParseCommandLineOptions(argc, argv, "llvm linker\n"); + // Load bitcode into the new debug info format by default. + if (LoadBitcodeIntoNewDbgInfoFormat == cl::boolOrDefault::BOU_UNSET) + LoadBitcodeIntoNewDbgInfoFormat = cl::boolOrDefault::BOU_TRUE; + // RemoveDIs debug-info transition: tests may request that we /try/ to use the // new debug-info format. if (TryUseNewDbgInfoFormat) { -- GitLab From 772e316457ef94759804d9f4da0af70d8d2ca4d4 Mon Sep 17 00:00:00 2001 From: Alexandros Lamprineas Date: Mon, 25 Mar 2024 09:43:41 +0000 Subject: [PATCH 198/306] [FMV] Allow multi versioning without default declaration. (#85454) This was a limitation which has now been lifted. Please read the thread below for more details: https://github.com/llvm/llvm-project/pull/84405#discussion_r1525583647 Basically it allows to separate versioned implementations across different TUs without having to share private header files which contain the default declaration. The ACLE spec has been updated accordingly to make this explicit: "Each version declaration should be visible at the translation unit in which the corresponding function version resides." https://github.com/ARM-software/acle/pull/310 If a resolver is required (because there is a caller in the TU), then a default declaration is implicitly generated. --- clang/lib/CodeGen/CodeGenModule.cpp | 131 ++++--- clang/lib/Sema/SemaDecl.cpp | 6 +- clang/lib/Sema/SemaOverload.cpp | 44 ++- clang/test/CodeGen/attr-target-version.c | 368 ++++++++++++------ clang/test/CodeGenCXX/attr-target-version.cpp | 62 +-- clang/test/Sema/aarch64-sme-func-attrs.c | 8 +- clang/test/Sema/attr-target-version.c | 8 +- clang/test/SemaCXX/attr-target-version.cpp | 11 +- 8 files changed, 403 insertions(+), 235 deletions(-) diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp index cb153066b28d..ac81df8cf7ad 100644 --- a/clang/lib/CodeGen/CodeGenModule.cpp +++ b/clang/lib/CodeGen/CodeGenModule.cpp @@ -3711,7 +3711,8 @@ void CodeGenModule::EmitGlobal(GlobalDecl GD) { // Forward declarations are emitted lazily on first use. if (!FD->doesThisDeclarationHaveABody()) { - if (!FD->doesDeclarationForceExternallyVisibleDefinition()) + if (!FD->doesDeclarationForceExternallyVisibleDefinition() && + !FD->isTargetVersionMultiVersion()) return; StringRef MangledName = getMangledName(GD); @@ -4092,6 +4093,23 @@ llvm::GlobalValue::LinkageTypes getMultiversionLinkage(CodeGenModule &CGM, return llvm::GlobalValue::WeakODRLinkage; } +static FunctionDecl *createDefaultTargetVersionFrom(const FunctionDecl *FD) { + DeclContext *DeclCtx = FD->getASTContext().getTranslationUnitDecl(); + TypeSourceInfo *TInfo = FD->getTypeSourceInfo(); + StorageClass SC = FD->getStorageClass(); + DeclarationName Name = FD->getNameInfo().getName(); + + FunctionDecl *NewDecl = + FunctionDecl::Create(FD->getASTContext(), DeclCtx, FD->getBeginLoc(), + FD->getEndLoc(), Name, TInfo->getType(), TInfo, SC); + + NewDecl->setIsMultiVersion(); + NewDecl->addAttr(TargetVersionAttr::CreateImplicit( + NewDecl->getASTContext(), "default", NewDecl->getSourceRange())); + + return NewDecl; +} + void CodeGenModule::emitMultiVersionFunctions() { std::vector MVFuncsToEmit; MultiVersionFuncs.swap(MVFuncsToEmit); @@ -4099,70 +4117,54 @@ void CodeGenModule::emitMultiVersionFunctions() { const auto *FD = cast(GD.getDecl()); assert(FD && "Expected a FunctionDecl"); - bool EmitResolver = !FD->isTargetVersionMultiVersion(); + auto createFunction = [&](const FunctionDecl *Decl, unsigned MVIdx = 0) { + GlobalDecl CurGD{Decl->isDefined() ? Decl->getDefinition() : Decl, MVIdx}; + StringRef MangledName = getMangledName(CurGD); + llvm::Constant *Func = GetGlobalValue(MangledName); + if (!Func) { + if (Decl->isDefined()) { + EmitGlobalFunctionDefinition(CurGD, nullptr); + Func = GetGlobalValue(MangledName); + } else { + const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(CurGD); + llvm::FunctionType *Ty = getTypes().GetFunctionType(FI); + Func = GetAddrOfFunction(CurGD, Ty, /*ForVTable=*/false, + /*DontDefer=*/false, ForDefinition); + } + assert(Func && "This should have just been created"); + } + return cast(Func); + }; + + bool HasDefaultDecl = !FD->isTargetVersionMultiVersion(); + bool ShouldEmitResolver = !FD->isTargetVersionMultiVersion(); SmallVector Options; if (FD->isTargetMultiVersion()) { getContext().forEachMultiversionedFunctionVersion( - FD, [this, &GD, &Options, &EmitResolver](const FunctionDecl *CurFD) { - GlobalDecl CurGD{ - (CurFD->isDefined() ? CurFD->getDefinition() : CurFD)}; - StringRef MangledName = getMangledName(CurGD); - llvm::Constant *Func = GetGlobalValue(MangledName); - if (!Func) { - if (CurFD->isDefined()) { - EmitGlobalFunctionDefinition(CurGD, nullptr); - Func = GetGlobalValue(MangledName); - } else { - const CGFunctionInfo &FI = - getTypes().arrangeGlobalDeclaration(GD); - llvm::FunctionType *Ty = getTypes().GetFunctionType(FI); - Func = GetAddrOfFunction(CurGD, Ty, /*ForVTable=*/false, - /*DontDefer=*/false, ForDefinition); - } - assert(Func && "This should have just been created"); - } - if (CurFD->getMultiVersionKind() == MultiVersionKind::Target) { - const auto *TA = CurFD->getAttr(); - llvm::SmallVector Feats; + FD, [&](const FunctionDecl *CurFD) { + llvm::SmallVector Feats; + llvm::Function *Func = createFunction(CurFD); + + if (const auto *TA = CurFD->getAttr()) { TA->getAddedFeatures(Feats); - Options.emplace_back(cast(Func), - TA->getArchitecture(), Feats); - } else { - const auto *TVA = CurFD->getAttr(); - if (CurFD->isUsed() || (TVA->isDefaultVersion() && - CurFD->doesThisDeclarationHaveABody())) - EmitResolver = true; - llvm::SmallVector Feats; + Options.emplace_back(Func, TA->getArchitecture(), Feats); + } else if (const auto *TVA = CurFD->getAttr()) { + bool HasDefaultDef = TVA->isDefaultVersion() && + CurFD->doesThisDeclarationHaveABody(); + HasDefaultDecl |= TVA->isDefaultVersion(); + ShouldEmitResolver |= (CurFD->isUsed() || HasDefaultDef); TVA->getFeatures(Feats); - Options.emplace_back(cast(Func), - /*Architecture*/ "", Feats); - } + Options.emplace_back(Func, /*Architecture*/ "", Feats); + } else + llvm_unreachable("unexpected MultiVersionKind"); }); - } else if (FD->isTargetClonesMultiVersion()) { - const auto *TC = FD->getAttr(); - for (unsigned VersionIndex = 0; VersionIndex < TC->featuresStrs_size(); - ++VersionIndex) { - if (!TC->isFirstOfVersion(VersionIndex)) + } else if (const auto *TC = FD->getAttr()) { + for (unsigned I = 0; I < TC->featuresStrs_size(); ++I) { + if (!TC->isFirstOfVersion(I)) continue; - GlobalDecl CurGD{(FD->isDefined() ? FD->getDefinition() : FD), - VersionIndex}; - StringRef Version = TC->getFeatureStr(VersionIndex); - StringRef MangledName = getMangledName(CurGD); - llvm::Constant *Func = GetGlobalValue(MangledName); - if (!Func) { - if (FD->isDefined()) { - EmitGlobalFunctionDefinition(CurGD, nullptr); - Func = GetGlobalValue(MangledName); - } else { - const CGFunctionInfo &FI = - getTypes().arrangeGlobalDeclaration(CurGD); - llvm::FunctionType *Ty = getTypes().GetFunctionType(FI); - Func = GetAddrOfFunction(CurGD, Ty, /*ForVTable=*/false, - /*DontDefer=*/false, ForDefinition); - } - assert(Func && "This should have just been created"); - } + llvm::Function *Func = createFunction(FD, I); + StringRef Version = TC->getFeatureStr(I); StringRef Architecture; llvm::SmallVector Feature; @@ -4180,16 +4182,23 @@ void CodeGenModule::emitMultiVersionFunctions() { Feature.push_back(Version); } - Options.emplace_back(cast(Func), Architecture, Feature); + Options.emplace_back(Func, Architecture, Feature); } } else { assert(0 && "Expected a target or target_clones multiversion function"); continue; } - if (!EmitResolver) + if (!ShouldEmitResolver) continue; + if (!HasDefaultDecl) { + FunctionDecl *NewFD = createDefaultTargetVersionFrom(FD); + llvm::Function *Func = createFunction(NewFD); + llvm::SmallVector Feats; + Options.emplace_back(Func, /*Architecture*/ "", Feats); + } + llvm::Constant *ResolverConstant = GetOrCreateMultiVersionResolver(GD); if (auto *IFunc = dyn_cast(ResolverConstant)) { ResolverConstant = IFunc->getResolver(); @@ -4480,7 +4489,9 @@ llvm::Constant *CodeGenModule::GetOrCreateLLVMFunction( if (FD->isMultiVersion()) { UpdateMultiVersionNames(GD, FD, MangledName); - if (!IsForDefinition) + if (FD->isTargetVersionMultiVersion() && !FD->isUsed()) + AddDeferredMultiVersionResolverToEmit(GD); + else if (!IsForDefinition) return GetOrCreateMultiVersionResolver(GD); } } diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp index aa754d47a0c4..73ea155053d7 100644 --- a/clang/lib/Sema/SemaDecl.cpp +++ b/clang/lib/Sema/SemaDecl.cpp @@ -11441,9 +11441,9 @@ static bool CheckMultiVersionFirstFunction(Sema &S, FunctionDecl *FD) { "Function lacks multiversion attribute"); const auto *TA = FD->getAttr(); const auto *TVA = FD->getAttr(); - // Target and target_version only causes MV if it is default, otherwise this - // is a normal function. - if ((TA && !TA->isDefaultVersion()) || (TVA && !TVA->isDefaultVersion())) + // The target attribute only causes MV if this declaration is the default, + // otherwise it is treated as a normal function. + if (TA && !TA->isDefaultVersion()) return false; if ((TA || TVA) && CheckMultiVersionValue(S, FD)) { diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp index f6bd85bdc646..51450e486eae 100644 --- a/clang/lib/Sema/SemaOverload.cpp +++ b/clang/lib/Sema/SemaOverload.cpp @@ -6865,6 +6865,32 @@ static bool IsAcceptableNonMemberOperatorCandidate(ASTContext &Context, return false; } +static bool isNonViableMultiVersionOverload(FunctionDecl *FD) { + if (FD->isTargetMultiVersionDefault()) + return false; + + if (!FD->getASTContext().getTargetInfo().getTriple().isAArch64()) + return FD->isTargetMultiVersion(); + + if (!FD->isMultiVersion()) + return false; + + // Among multiple target versions consider either the default, + // or the first non-default in the absence of default version. + unsigned SeenAt = 0; + unsigned I = 0; + bool HasDefault = false; + FD->getASTContext().forEachMultiversionedFunctionVersion( + FD, [&](const FunctionDecl *CurFD) { + if (FD == CurFD) + SeenAt = I; + else if (CurFD->isTargetMultiVersionDefault()) + HasDefault = true; + ++I; + }); + return HasDefault || SeenAt != 0; +} + /// AddOverloadCandidate - Adds the given function to the set of /// candidate functions, using the given function call arguments. If /// @p SuppressUserConversions, then don't allow user-defined @@ -6970,11 +6996,7 @@ void Sema::AddOverloadCandidate( } } - if (Function->isMultiVersion() && - ((Function->hasAttr() && - !Function->getAttr()->isDefaultVersion()) || - (Function->hasAttr() && - !Function->getAttr()->isDefaultVersion()))) { + if (isNonViableMultiVersionOverload(Function)) { Candidate.Viable = false; Candidate.FailureKind = ovl_non_default_multiversion_function; return; @@ -7637,11 +7659,7 @@ Sema::AddMethodCandidate(CXXMethodDecl *Method, DeclAccessPair FoundDecl, return; } - if (Method->isMultiVersion() && - ((Method->hasAttr() && - !Method->getAttr()->isDefaultVersion()) || - (Method->hasAttr() && - !Method->getAttr()->isDefaultVersion()))) { + if (isNonViableMultiVersionOverload(Method)) { Candidate.Viable = false; Candidate.FailureKind = ovl_non_default_multiversion_function; } @@ -8127,11 +8145,7 @@ void Sema::AddConversionCandidate( return; } - if (Conversion->isMultiVersion() && - ((Conversion->hasAttr() && - !Conversion->getAttr()->isDefaultVersion()) || - (Conversion->hasAttr() && - !Conversion->getAttr()->isDefaultVersion()))) { + if (isNonViableMultiVersionOverload(Conversion)) { Candidate.Viable = false; Candidate.FailureKind = ovl_non_default_multiversion_function; } diff --git a/clang/test/CodeGen/attr-target-version.c b/clang/test/CodeGen/attr-target-version.c index 25129605e76c..dd4cbbf5a898 100644 --- a/clang/test/CodeGen/attr-target-version.c +++ b/clang/test/CodeGen/attr-target-version.c @@ -109,21 +109,47 @@ int unused_with_implicit_default_def(void) { return 1; } int unused_with_implicit_forward_default_def(void) { return 0; } __attribute__((target_version("lse"))) int unused_with_implicit_forward_default_def(void) { return 1; } -// This should generate a normal function. +// This should generate a target version despite the default not being declared. __attribute__((target_version("rdm"))) int unused_without_default(void) { return 0; } +// These shouldn't generate anything. +int unused_version_declarations(void); +__attribute__((target_version("jscvt"))) int unused_version_declarations(void); +__attribute__((target_version("rdma"))) int unused_version_declarations(void); + +// These should generate the default (mangled) version and the resolver. +int default_def_with_version_decls(void) { return 0; } +__attribute__((target_version("jscvt"))) int default_def_with_version_decls(void); +__attribute__((target_version("rdma"))) int default_def_with_version_decls(void); + +// The following is guarded because in NOFMV we get errors for calling undeclared functions. +#ifdef __HAVE_FUNCTION_MULTI_VERSIONING +// This should generate a default declaration, two target versions and the resolver. +__attribute__((target_version("jscvt"))) int used_def_without_default_decl(void) { return 1; } +__attribute__((target_version("rdma"))) int used_def_without_default_decl(void) { return 2; } + +// This should generate a default declaration and the resolver. +__attribute__((target_version("jscvt"))) int used_decl_without_default_decl(void); +__attribute__((target_version("rdma"))) int used_decl_without_default_decl(void); + +int caller(void) { return used_def_without_default_decl() + used_decl_without_default_decl(); } +#endif + //. // CHECK: @__aarch64_cpu_features = external dso_local global { i64 } // CHECK: @fmv.ifunc = weak_odr alias i32 (), ptr @fmv // CHECK: @fmv_one.ifunc = weak_odr alias i32 (), ptr @fmv_one // CHECK: @fmv_two.ifunc = weak_odr alias i32 (), ptr @fmv_two // CHECK: @fmv_e.ifunc = weak_odr alias i32 (), ptr @fmv_e +// CHECK: @fmv_d.ifunc = internal alias i32 (), ptr @fmv_d // CHECK: @fmv_c.ifunc = weak_odr alias void (), ptr @fmv_c // CHECK: @fmv_inline.ifunc = weak_odr alias i32 (), ptr @fmv_inline -// CHECK: @fmv_d.ifunc = internal alias i32 (), ptr @fmv_d // CHECK: @unused_with_default_def.ifunc = weak_odr alias i32 (), ptr @unused_with_default_def // CHECK: @unused_with_implicit_default_def.ifunc = weak_odr alias i32 (), ptr @unused_with_implicit_default_def // CHECK: @unused_with_implicit_forward_default_def.ifunc = weak_odr alias i32 (), ptr @unused_with_implicit_forward_default_def +// CHECK: @default_def_with_version_decls.ifunc = weak_odr alias i32 (), ptr @default_def_with_version_decls +// CHECK: @used_def_without_default_decl.ifunc = weak_odr alias i32 (), ptr @used_def_without_default_decl +// CHECK: @used_decl_without_default_decl.ifunc = weak_odr alias i32 (), ptr @used_decl_without_default_decl // CHECK: @fmv = weak_odr ifunc i32 (), ptr @fmv.resolver // CHECK: @fmv_one = weak_odr ifunc i32 (), ptr @fmv_one.resolver // CHECK: @fmv_two = weak_odr ifunc i32 (), ptr @fmv_two.resolver @@ -131,97 +157,121 @@ __attribute__((target_version("rdm"))) int unused_without_default(void) { return // CHECK: @fmv_e = weak_odr ifunc i32 (), ptr @fmv_e.resolver // CHECK: @fmv_d = internal ifunc i32 (), ptr @fmv_d.resolver // CHECK: @fmv_c = weak_odr ifunc void (), ptr @fmv_c.resolver +// CHECK: @used_def_without_default_decl = weak_odr ifunc i32 (), ptr @used_def_without_default_decl.resolver +// CHECK: @used_decl_without_default_decl = weak_odr ifunc i32 (), ptr @used_decl_without_default_decl.resolver // CHECK: @unused_with_default_def = weak_odr ifunc i32 (), ptr @unused_with_default_def.resolver // CHECK: @unused_with_implicit_default_def = weak_odr ifunc i32 (), ptr @unused_with_implicit_default_def.resolver // CHECK: @unused_with_implicit_forward_default_def = weak_odr ifunc i32 (), ptr @unused_with_implicit_forward_default_def.resolver +// CHECK: @default_def_with_version_decls = weak_odr ifunc i32 (), ptr @default_def_with_version_decls.resolver //. // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv._Mflagm2Msme-i16i64 +// CHECK-LABEL: define {{[^@]+}}@fmv._MflagmMfp16fmlMrng // CHECK-SAME: () #[[ATTR0:[0-9]+]] { // CHECK-NEXT: entry: +// CHECK-NEXT: ret i32 1 +// +// +// CHECK: Function Attrs: noinline nounwind optnone +// CHECK-LABEL: define {{[^@]+}}@fmv._Mflagm2Msme-i16i64 +// CHECK-SAME: () #[[ATTR1:[0-9]+]] { +// CHECK-NEXT: entry: // CHECK-NEXT: ret i32 2 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv._MlseMsha2 -// CHECK-SAME: () #[[ATTR1:[0-9]+]] { +// CHECK-SAME: () #[[ATTR2:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 3 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv._MdotprodMls64_accdata -// CHECK-SAME: () #[[ATTR2:[0-9]+]] { +// CHECK-SAME: () #[[ATTR3:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 4 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv._Mfp16fmlMmemtag -// CHECK-SAME: () #[[ATTR3:[0-9]+]] { +// CHECK-SAME: () #[[ATTR4:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 5 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv._MaesMfp -// CHECK-SAME: () #[[ATTR4:[0-9]+]] { +// CHECK-SAME: () #[[ATTR5:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 6 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv._McrcMls64_v -// CHECK-SAME: () #[[ATTR5:[0-9]+]] { +// CHECK-SAME: () #[[ATTR6:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 7 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv._Mbti -// CHECK-SAME: () #[[ATTR6:[0-9]+]] { +// CHECK-SAME: () #[[ATTR7:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 8 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv._Msme2 -// CHECK-SAME: () #[[ATTR7:[0-9]+]] { +// CHECK-SAME: () #[[ATTR8:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 9 // // // CHECK: Function Attrs: noinline nounwind optnone +// CHECK-LABEL: define {{[^@]+}}@fmv_one._Mls64Msimd +// CHECK-SAME: () #[[ATTR5]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: ret i32 1 +// +// +// CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_one._Mdpb -// CHECK-SAME: () #[[ATTR8:[0-9]+]] { +// CHECK-SAME: () #[[ATTR10:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 2 // // // CHECK: Function Attrs: noinline nounwind optnone +// CHECK-LABEL: define {{[^@]+}}@fmv_two._Mfp +// CHECK-SAME: () #[[ATTR5]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: ret i32 1 +// +// +// CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_two._Msimd -// CHECK-SAME: () #[[ATTR4]] { +// CHECK-SAME: () #[[ATTR5]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 2 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_two._Mdgh -// CHECK-SAME: () #[[ATTR9:[0-9]+]] { +// CHECK-SAME: () #[[ATTR11:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 3 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_two._Mfp16Msimd -// CHECK-SAME: () #[[ATTR10:[0-9]+]] { +// CHECK-SAME: () #[[ATTR12:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 4 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@foo -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: [[CALL:%.*]] = call i32 @fmv() // CHECK-NEXT: [[CALL1:%.*]] = call i32 @fmv_one() @@ -371,35 +421,49 @@ __attribute__((target_version("rdm"))) int unused_without_default(void) { return // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_e.default -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 20 // // // CHECK: Function Attrs: noinline nounwind optnone +// CHECK-LABEL: define {{[^@]+}}@fmv_d._Msb +// CHECK-SAME: () #[[ATTR13:[0-9]+]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: ret i32 0 +// +// +// CHECK: Function Attrs: noinline nounwind optnone +// CHECK-LABEL: define {{[^@]+}}@fmv_d.default +// CHECK-SAME: () #[[ATTR11]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: ret i32 1 +// +// +// CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_default -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 111 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_c._Mssbs -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret void // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_c.default -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret void // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@goo -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: [[CALL:%.*]] = call i32 @fmv_inline() // CHECK-NEXT: [[CALL1:%.*]] = call i32 @fmv_e() @@ -587,7 +651,7 @@ __attribute__((target_version("rdm"))) int unused_without_default(void) { return // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@recur -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: call void @reca() // CHECK-NEXT: ret void @@ -595,7 +659,7 @@ __attribute__((target_version("rdm"))) int unused_without_default(void) { return // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@main -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: [[RETVAL:%.*]] = alloca i32, align 4 // CHECK-NEXT: store i32 0, ptr [[RETVAL]], align 4 @@ -606,7 +670,7 @@ __attribute__((target_version("rdm"))) int unused_without_default(void) { return // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@hoo -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: [[FP1:%.*]] = alloca ptr, align 8 // CHECK-NEXT: [[FP2:%.*]] = alloca ptr, align 8 @@ -623,228 +687,268 @@ __attribute__((target_version("rdm"))) int unused_without_default(void) { return // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@unused_with_forward_default_decl._Mmops -// CHECK-SAME: () #[[ATTR12:[0-9]+]] { +// CHECK-SAME: () #[[ATTR14:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 0 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_extern_forward_default_decl._Mdotprod -// CHECK-SAME: () #[[ATTR13:[0-9]+]] { +// CHECK-SAME: () #[[ATTR15:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 0 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@unused_with_default_def.default -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-LABEL: define {{[^@]+}}@unused_with_default_decl._Maes +// CHECK-SAME: () #[[ATTR5]] { // CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 1 +// CHECK-NEXT: ret i32 0 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_default_def.default -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-LABEL: define {{[^@]+}}@unused_with_default_def._Msve +// CHECK-SAME: () #[[ATTR16:[0-9]+]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: ret i32 0 +// +// +// CHECK: Function Attrs: noinline nounwind optnone +// CHECK-LABEL: define {{[^@]+}}@unused_with_default_def.default +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 1 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_forward_default_def.default -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_default_def._Mfp16 +// CHECK-SAME: () #[[ATTR12]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 0 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_forward_default_def._Mlse -// CHECK-SAME: () #[[ATTR14:[0-9]+]] { +// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_default_def.default +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 1 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv._MflagmMfp16fmlMrng -// CHECK-SAME: () #[[ATTR15:[0-9]+]] { +// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_forward_default_def.default +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 1 +// CHECK-NEXT: ret i32 0 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv_one._Mls64Msimd -// CHECK-SAME: () #[[ATTR4]] { +// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_forward_default_def._Mlse +// CHECK-SAME: () #[[ATTR17:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 1 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv_two._Mfp -// CHECK-SAME: () #[[ATTR4]] { +// CHECK-LABEL: define {{[^@]+}}@unused_without_default._Mrdm +// CHECK-SAME: () #[[ATTR18:[0-9]+]] { // CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 1 +// CHECK-NEXT: ret i32 0 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@unused_with_default_decl._Maes -// CHECK-SAME: () #[[ATTR4]] { +// CHECK-LABEL: define {{[^@]+}}@default_def_with_version_decls.default +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 0 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@unused_with_default_def._Msve -// CHECK-SAME: () #[[ATTR16:[0-9]+]] { +// CHECK-LABEL: define {{[^@]+}}@used_def_without_default_decl._Mjscvt +// CHECK-SAME: () #[[ATTR21:[0-9]+]] { // CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 0 +// CHECK-NEXT: ret i32 1 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_default_def._Mfp16 -// CHECK-SAME: () #[[ATTR10]] { +// CHECK-LABEL: define {{[^@]+}}@used_def_without_default_decl._Mrdm +// CHECK-SAME: () #[[ATTR18]] { // CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 0 +// CHECK-NEXT: ret i32 2 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@unused_without_default -// CHECK-SAME: () #[[ATTR17:[0-9]+]] { +// CHECK-LABEL: define {{[^@]+}}@caller +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 0 +// CHECK-NEXT: [[CALL:%.*]] = call i32 @used_def_without_default_decl() +// CHECK-NEXT: [[CALL1:%.*]] = call i32 @used_decl_without_default_decl() +// CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[CALL]], [[CALL1]] +// CHECK-NEXT: ret i32 [[ADD]] +// +// +// CHECK-LABEL: define {{[^@]+}}@used_def_without_default_decl.resolver() comdat { +// CHECK-NEXT: resolver_entry: +// CHECK-NEXT: call void @__init_cpu_features_resolver() +// CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 1048576 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1048576 +// CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] +// CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] +// CHECK: resolver_return: +// CHECK-NEXT: ret ptr @used_def_without_default_decl._Mjscvt +// CHECK: resolver_else: +// CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 +// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 64 +// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 64 +// CHECK-NEXT: [[TMP7:%.*]] = and i1 true, [[TMP6]] +// CHECK-NEXT: br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]] +// CHECK: resolver_return1: +// CHECK-NEXT: ret ptr @used_def_without_default_decl._Mrdm +// CHECK: resolver_else2: +// CHECK-NEXT: ret ptr @used_def_without_default_decl.default +// +// +// CHECK-LABEL: define {{[^@]+}}@used_decl_without_default_decl.resolver() comdat { +// CHECK-NEXT: resolver_entry: +// CHECK-NEXT: call void @__init_cpu_features_resolver() +// CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 1048576 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1048576 +// CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] +// CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] +// CHECK: resolver_return: +// CHECK-NEXT: ret ptr @used_decl_without_default_decl._Mjscvt +// CHECK: resolver_else: +// CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 +// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 64 +// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 64 +// CHECK-NEXT: [[TMP7:%.*]] = and i1 true, [[TMP6]] +// CHECK-NEXT: br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]] +// CHECK: resolver_return1: +// CHECK-NEXT: ret ptr @used_decl_without_default_decl._Mrdm +// CHECK: resolver_else2: +// CHECK-NEXT: ret ptr @used_decl_without_default_decl.default // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mf64mmMpmullMsha1 -// CHECK-SAME: () #[[ATTR18:[0-9]+]] { +// CHECK-SAME: () #[[ATTR22:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 1 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MfcmaMfp16MrdmMsme -// CHECK-SAME: () #[[ATTR19:[0-9]+]] { +// CHECK-SAME: () #[[ATTR23:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 2 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mf32mmMi8mmMsha3 -// CHECK-SAME: () #[[ATTR20:[0-9]+]] { +// CHECK-SAME: () #[[ATTR24:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 12 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MditMsve-ebf16 -// CHECK-SAME: () #[[ATTR21:[0-9]+]] { +// CHECK-SAME: () #[[ATTR25:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 8 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MdpbMrcpc2 -// CHECK-SAME: () #[[ATTR22:[0-9]+]] { +// CHECK-SAME: () #[[ATTR26:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 6 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mdpb2Mjscvt -// CHECK-SAME: () #[[ATTR23:[0-9]+]] { +// CHECK-SAME: () #[[ATTR27:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 7 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MfrinttsMrcpc -// CHECK-SAME: () #[[ATTR24:[0-9]+]] { +// CHECK-SAME: () #[[ATTR28:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 3 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MsveMsve-bf16 -// CHECK-SAME: () #[[ATTR25:[0-9]+]] { +// CHECK-SAME: () #[[ATTR29:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 4 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Msve2-aesMsve2-sha3 -// CHECK-SAME: () #[[ATTR26:[0-9]+]] { +// CHECK-SAME: () #[[ATTR30:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 5 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Msve2Msve2-bitpermMsve2-pmull128 -// CHECK-SAME: () #[[ATTR27:[0-9]+]] { +// CHECK-SAME: () #[[ATTR31:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 9 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mmemtag2Msve2-sm4 -// CHECK-SAME: () #[[ATTR28:[0-9]+]] { +// CHECK-SAME: () #[[ATTR32:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 10 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mmemtag3MmopsMrcpc3 -// CHECK-SAME: () #[[ATTR29:[0-9]+]] { +// CHECK-SAME: () #[[ATTR33:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 11 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MaesMdotprod -// CHECK-SAME: () #[[ATTR13]] { +// CHECK-SAME: () #[[ATTR15]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 13 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mfp16fmlMsimd -// CHECK-SAME: () #[[ATTR3]] { +// CHECK-SAME: () #[[ATTR4]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 14 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MfpMsm4 -// CHECK-SAME: () #[[ATTR30:[0-9]+]] { +// CHECK-SAME: () #[[ATTR34:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 15 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MlseMrdm -// CHECK-SAME: () #[[ATTR31:[0-9]+]] { +// CHECK-SAME: () #[[ATTR35:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 16 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline.default -// CHECK-SAME: () #[[ATTR9]] { +// CHECK-SAME: () #[[ATTR11]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 3 // // -// CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv_d._Msb -// CHECK-SAME: () #[[ATTR32:[0-9]+]] { -// CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 0 -// -// -// CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv_d.default -// CHECK-SAME: () #[[ATTR9]] { -// CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 1 -// -// // CHECK-LABEL: define {{[^@]+}}@unused_with_default_def.resolver() comdat { // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() @@ -887,6 +991,28 @@ __attribute__((target_version("rdm"))) int unused_without_default(void) { return // CHECK-NEXT: ret ptr @unused_with_implicit_forward_default_def.default // // +// CHECK-LABEL: define {{[^@]+}}@default_def_with_version_decls.resolver() comdat { +// CHECK-NEXT: resolver_entry: +// CHECK-NEXT: call void @__init_cpu_features_resolver() +// CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 1048576 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1048576 +// CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] +// CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] +// CHECK: resolver_return: +// CHECK-NEXT: ret ptr @default_def_with_version_decls._Mjscvt +// CHECK: resolver_else: +// CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 +// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 64 +// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 64 +// CHECK-NEXT: [[TMP7:%.*]] = and i1 true, [[TMP6]] +// CHECK-NEXT: br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]] +// CHECK: resolver_return1: +// CHECK-NEXT: ret ptr @default_def_with_version_decls._Mrdm +// CHECK: resolver_else2: +// CHECK-NEXT: ret ptr @default_def_with_version_decls.default +// +// // CHECK-NOFMV: Function Attrs: noinline nounwind optnone // CHECK-NOFMV-LABEL: define {{[^@]+}}@foo // CHECK-NOFMV-SAME: () #[[ATTR0:[0-9]+]] { @@ -995,40 +1121,50 @@ __attribute__((target_version("rdm"))) int unused_without_default(void) { return // CHECK-NOFMV-NEXT: entry: // CHECK-NOFMV-NEXT: ret i32 0 // +// +// CHECK-NOFMV: Function Attrs: noinline nounwind optnone +// CHECK-NOFMV-LABEL: define {{[^@]+}}@default_def_with_version_decls +// CHECK-NOFMV-SAME: () #[[ATTR0]] { +// CHECK-NOFMV-NEXT: entry: +// CHECK-NOFMV-NEXT: ret i32 0 +// //. -// CHECK: attributes #[[ATTR0]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+altnzcv,+bf16,+flagm,+sme,+sme-i16i64,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR1]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,+neon,+sha2,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR2]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+ls64,+neon,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR3]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp16fml,+fullfp16,+neon,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR4]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR5]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+crc,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR6]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bti,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR7]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+sme,+sme2,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR8]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccpp,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR9]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR10]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR11:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR12]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR13]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+neon,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR14]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR15]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+flagm,+fp16fml,+fullfp16,+neon,+rand,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR0]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+flagm,+fp16fml,+fullfp16,+neon,+rand,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR1]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+altnzcv,+bf16,+flagm,+sme,+sme-i16i64,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR2]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,+neon,+sha2,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR3]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+ls64,+neon,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR4]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp16fml,+fullfp16,+neon,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR5]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR6]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+crc,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR7]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bti,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR8]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+sme,+sme2,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR9:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR10]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccpp,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR11]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR12]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR13]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+sb,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR14]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR15]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+neon,-fp-armv8,-v9.5a" } // CHECK: attributes #[[ATTR16]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR17]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,+rdm,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR18]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+aes,+f64mm,+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR19]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+complxnum,+fullfp16,+neon,+rdm,+sme,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR20]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+f32mm,+fullfp16,+i8mm,+neon,+sha2,+sha3,+sve,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR21]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+dit,+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR22]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccpp,+rcpc,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR23]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccdp,+ccpp,+jsconv,+neon,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR24]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fptoint,+rcpc,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR25]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR26]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,+sve,+sve2,+sve2-aes,+sve2-sha3,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR27]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,+sve,+sve2,+sve2-aes,+sve2-bitperm,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR28]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+mte,+neon,+sve,+sve2,+sve2-sm4,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR29]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops,+mte,+rcpc,+rcpc3,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR30]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,+sm4,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR31]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,+neon,+rdm,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR32]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+sb,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR17]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR18]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,+rdm,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR19:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+jsconv,+neon,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR20:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,+rdm,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR21]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+jsconv,+neon,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR22]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+aes,+f64mm,+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR23]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+complxnum,+fullfp16,+neon,+rdm,+sme,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR24]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+f32mm,+fullfp16,+i8mm,+neon,+sha2,+sha3,+sve,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR25]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+dit,+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR26]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccpp,+rcpc,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR27]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccdp,+ccpp,+jsconv,+neon,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR28]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fptoint,+rcpc,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR29]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR30]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,+sve,+sve2,+sve2-aes,+sve2-sha3,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR31]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,+sve,+sve2,+sve2-aes,+sve2-bitperm,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR32]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+mte,+neon,+sve,+sve2,+sve2-sm4,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR33]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops,+mte,+rcpc,+rcpc3,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR34]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,+sm4,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR35]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,+neon,+rdm,-fp-armv8,-v9.5a" } //. // CHECK-NOFMV: attributes #[[ATTR0]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fmv" } // CHECK-NOFMV: attributes #[[ATTR1:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fmv" } diff --git a/clang/test/CodeGenCXX/attr-target-version.cpp b/clang/test/CodeGenCXX/attr-target-version.cpp index e06121d1a719..8b7273fe3bb5 100644 --- a/clang/test/CodeGenCXX/attr-target-version.cpp +++ b/clang/test/CodeGenCXX/attr-target-version.cpp @@ -35,7 +35,7 @@ struct MyClass { int unused_with_implicit_forward_default_def(void); int __attribute__((target_version("lse"))) unused_with_implicit_forward_default_def(void); - // This should generate a normal function. + // This should generate a target version despite the default not being declared. int __attribute__((target_version("rdm"))) unused_without_default(void); }; @@ -75,6 +75,13 @@ int bar() { // CHECK: @_ZN7MyClass32unused_with_implicit_default_defEv = weak_odr ifunc i32 (ptr), ptr @_ZN7MyClass32unused_with_implicit_default_defEv.resolver // CHECK: @_ZN7MyClass40unused_with_implicit_forward_default_defEv = weak_odr ifunc i32 (ptr), ptr @_ZN7MyClass40unused_with_implicit_forward_default_defEv.resolver //. +// CHECK-LABEL: @_Z3fooi._Mbf16Msme-f64f64( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[DOTADDR:%.*]] = alloca i32, align 4 +// CHECK-NEXT: store i32 [[TMP0:%.*]], ptr [[DOTADDR]], align 4 +// CHECK-NEXT: ret i32 1 +// +// // CHECK-LABEL: @_Z3fooi.default( // CHECK-NEXT: entry: // CHECK-NEXT: [[DOTADDR:%.*]] = alloca i32, align 4 @@ -82,6 +89,11 @@ int bar() { // CHECK-NEXT: ret i32 2 // // +// CHECK-LABEL: @_Z3foov._Mebf16Msm4( +// CHECK-NEXT: entry: +// CHECK-NEXT: ret i32 3 +// +// // CHECK-LABEL: @_Z3foov.default( // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 4 @@ -189,6 +201,14 @@ int bar() { // CHECK-NEXT: ret i32 1 // // +// CHECK-LABEL: @_ZN7MyClass22unused_without_defaultEv._Mrdm( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[THIS_ADDR:%.*]] = alloca ptr, align 8 +// CHECK-NEXT: store ptr [[THIS:%.*]], ptr [[THIS_ADDR]], align 8 +// CHECK-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8 +// CHECK-NEXT: ret i32 0 +// +// // CHECK-LABEL: @_Z3barv( // CHECK-NEXT: entry: // CHECK-NEXT: [[M:%.*]] = alloca [[STRUCT_MYCLASS:%.*]], align 1 @@ -250,26 +270,6 @@ int bar() { // CHECK-NEXT: ret ptr @_Z3foov.default // // -// CHECK-LABEL: @_Z3fooi._Mbf16Msme-f64f64( -// CHECK-NEXT: entry: -// CHECK-NEXT: [[DOTADDR:%.*]] = alloca i32, align 4 -// CHECK-NEXT: store i32 [[TMP0:%.*]], ptr [[DOTADDR]], align 4 -// CHECK-NEXT: ret i32 1 -// -// -// CHECK-LABEL: @_Z3foov._Mebf16Msm4( -// CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 3 -// -// -// CHECK-LABEL: @_ZN7MyClass22unused_without_defaultEv( -// CHECK-NEXT: entry: -// CHECK-NEXT: [[THIS_ADDR:%.*]] = alloca ptr, align 8 -// CHECK-NEXT: store ptr [[THIS:%.*]], ptr [[THIS_ADDR]], align 8 -// CHECK-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8 -// CHECK-NEXT: ret i32 0 -// -// // CHECK-LABEL: @_ZN7MyClass23unused_with_default_defEv.resolver( // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() @@ -312,16 +312,16 @@ int bar() { // CHECK-NEXT: ret ptr @_ZN7MyClass40unused_with_implicit_forward_default_defEv.default // //. -// CHECK: attributes #[[ATTR0:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" } -// CHECK: attributes #[[ATTR1:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+crc" } -// CHECK: attributes #[[ATTR2:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+fp-armv8,+neon" } -// CHECK: attributes #[[ATTR3:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops" } -// CHECK: attributes #[[ATTR4:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon" } -// CHECK: attributes #[[ATTR5:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon,+sve" } -// CHECK: attributes #[[ATTR6:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon" } -// CHECK: attributes #[[ATTR7:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse" } -// CHECK: attributes #[[ATTR8:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+sme,+sme-f64f64" } -// CHECK: attributes #[[ATTR9:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+fp-armv8,+neon,+sm4" } +// CHECK: attributes #[[ATTR0:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+sme,+sme-f64f64" } +// CHECK: attributes #[[ATTR1:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +// CHECK: attributes #[[ATTR2:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+fp-armv8,+neon,+sm4" } +// CHECK: attributes #[[ATTR3:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+crc" } +// CHECK: attributes #[[ATTR4:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+fp-armv8,+neon" } +// CHECK: attributes #[[ATTR5:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops" } +// CHECK: attributes #[[ATTR6:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon" } +// CHECK: attributes #[[ATTR7:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon,+sve" } +// CHECK: attributes #[[ATTR8:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon" } +// CHECK: attributes #[[ATTR9:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse" } // CHECK: attributes #[[ATTR10:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon,+rdm" } // CHECK: attributes #[[ATTR11:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" } //. diff --git a/clang/test/Sema/aarch64-sme-func-attrs.c b/clang/test/Sema/aarch64-sme-func-attrs.c index 47dbeca206a9..bfc8768c3f36 100644 --- a/clang/test/Sema/aarch64-sme-func-attrs.c +++ b/clang/test/Sema/aarch64-sme-func-attrs.c @@ -483,14 +483,16 @@ void just_fine(void) {} __arm_locally_streaming __attribute__((target_version("sme2"))) -void just_fine_locally_streaming(void) {} +void incompatible_locally_streaming(void) {} +// expected-error@-1 {{attribute 'target_version' multiversioning cannot be combined with attribute '__arm_locally_streaming'}} +// expected-cpp-error@-2 {{attribute 'target_version' multiversioning cannot be combined with attribute '__arm_locally_streaming'}} __attribute__((target_version("default"))) -void just_fine_locally_streaming(void) {} +void incompatible_locally_streaming(void) {} void fmv_caller() { cannot_work_version(); cannot_work_clones(); just_fine(); - just_fine_locally_streaming(); + incompatible_locally_streaming(); } diff --git a/clang/test/Sema/attr-target-version.c b/clang/test/Sema/attr-target-version.c index e2940c434c2f..cd5be459456e 100644 --- a/clang/test/Sema/attr-target-version.c +++ b/clang/test/Sema/attr-target-version.c @@ -68,13 +68,15 @@ int __attribute__((target_version(""))) unsup1(void) { return 1; } void __attribute__((target_version("crc32"))) unsup2(void) {} void __attribute__((target_version("default+fp16"))) koo(void) {} +//expected-error@-1 {{function multiversioning doesn't support feature 'default'}} void __attribute__((target_version("default+default+default"))) loo(void) {} +//expected-error@-1 {{function multiversioning doesn't support feature 'default'}} void __attribute__((target_version("rdm+rng+crc"))) redef(void) {} //expected-error@+2 {{redefinition of 'redef'}} //expected-note@-2 {{previous definition is here}} void __attribute__((target_version("rdm+rng+crc"))) redef(void) {} -int __attribute__((target_version("sm4"))) def(void); +int def(void); void __attribute__((target_version("dit"))) nodef(void); void __attribute__((target_version("ls64"))) nodef(void); void __attribute__((target_version("aes"))) ovl(void); @@ -83,7 +85,6 @@ int bar() { // expected-error@+2 {{reference to overloaded function could not be resolved; did you mean to call it?}} // expected-note@-3 {{possible target for call}} ovl++; - // expected-error@+1 {{no matching function for call to 'nodef'}} nodef(); return def(); } @@ -92,8 +93,6 @@ int __attribute__((target_version("sha1"))) def(void) { return 1; } int __attribute__((target_version("sve"))) prot(); // expected-error@-1 {{multiversioned function must have a prototype}} -// expected-note@+1 {{function multiversioning caused by this declaration}} -int __attribute__((target_version("fcma"))) prot(); int __attribute__((target_version("pmull"))) rtype(int); // expected-error@+1 {{multiversioned function declaration has a different return type}} @@ -104,6 +103,7 @@ int __attribute__((target_version("sha2"))) combine(void) { return 1; } int __attribute__((aarch64_vector_pcs, target_version("sha3"))) combine(void) { return 2; } int __attribute__((target_version("fp+aes+pmull+rcpc"))) unspec_args() { return -1; } +// expected-error@-1 {{multiversioned function must have a prototype}} // expected-error@+1 {{multiversioned function must have a prototype}} int __attribute__((target_version("default"))) unspec_args() { return 0; } int cargs() { return unspec_args(); } diff --git a/clang/test/SemaCXX/attr-target-version.cpp b/clang/test/SemaCXX/attr-target-version.cpp index 0bd710c4e282..b3385f043590 100644 --- a/clang/test/SemaCXX/attr-target-version.cpp +++ b/clang/test/SemaCXX/attr-target-version.cpp @@ -9,7 +9,6 @@ void __attribute__((target_version("rcpc3"))) no_def(void); void __attribute__((target_version("mops"))) no_def(void); void __attribute__((target_version("rdma"))) no_def(void); -// expected-error@+1 {{no matching function for call to 'no_def'}} void foo(void) { no_def(); } constexpr int __attribute__((target_version("sve2"))) diff_const(void) { return 1; } @@ -41,6 +40,7 @@ inline int __attribute__((target_version("sme"))) diff_inline(void) { return 1; int __attribute__((target_version("fp16"))) diff_inline(void) { return 2; } inline int __attribute__((target_version("sme"))) diff_inline1(void) { return 1; } +//expected-error@+1 {{multiversioned function declaration has a different inline specification}} int __attribute__((target_version("default"))) diff_inline1(void) { return 2; } int __attribute__((target_version("fcma"))) diff_type1(void) { return 1; } @@ -59,8 +59,7 @@ int __attribute__((target_version("sve2-sha3"))) diff_type3(void) noexcept(true) template int __attribute__((target_version("default"))) temp(T) { return 1; } template int __attribute__((target_version("simd"))) temp1(T) { return 1; } -// expected-error@+1 {{attribute 'target_version' multiversioned functions do not yet support function templates}} -template int __attribute__((target_version("sha3"))) temp1(T) { return 2; } +// expected-error@-1 {{attribute 'target_version' multiversioned functions do not yet support function templates}} extern "C" { int __attribute__((target_version("aes"))) extc(void) { return 1; } @@ -70,17 +69,23 @@ int __attribute__((target_version("lse"))) extc(void) { return 1; } auto __attribute__((target_version("default"))) ret1(void) { return 1; } auto __attribute__((target_version("dpb"))) ret2(void) { return 1; } +// expected-error@-1 {{attribute 'target_version' multiversioned functions do not yet support deduced return types}} auto __attribute__((target_version("dpb2"))) ret3(void) -> int { return 1; } class Cls { __attribute__((target_version("rng"))) Cls(); + // expected-error@-1 {{attribute 'target_version' multiversioned functions do not yet support constructors}} __attribute__((target_version("sve-i8mm"))) ~Cls(); + // expected-error@-1 {{attribute 'target_version' multiversioned functions do not yet support destructors}} Cls &__attribute__((target_version("f32mm"))) operator=(const Cls &) = default; + // expected-error@-1 {{attribute 'target_version' multiversioned functions do not yet support defaulted functions}} Cls &__attribute__((target_version("ssbs"))) operator=(Cls &&) = delete; + // expected-error@-1 {{attribute 'target_version' multiversioned functions do not yet support deleted functions}} virtual void __attribute__((target_version("default"))) vfunc(); virtual void __attribute__((target_version("sm4"))) vfunc1(); + // expected-error@-1 {{attribute 'target_version' multiversioned functions do not yet support virtual functions}} }; __attribute__((target_version("sha3"))) void Decl(); -- GitLab From 7434a6b96c68b6835a6de6e98199dcfc35981dd5 Mon Sep 17 00:00:00 2001 From: pvanhout Date: Mon, 25 Mar 2024 10:50:08 +0100 Subject: [PATCH 199/306] [TableGen] Fix Linker Errors Fix linker errors after landing fa3d789df15bd1f58fb8ba4ea3be909218cf7f03 --- llvm/utils/TableGen/Basic/CMakeLists.txt | 2 +- llvm/utils/TableGen/CMakeLists.txt | 2 +- llvm/utils/TableGen/Common/CMakeLists.txt | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/llvm/utils/TableGen/Basic/CMakeLists.txt b/llvm/utils/TableGen/Basic/CMakeLists.txt index f2927d05c175..5a899e3b7c80 100644 --- a/llvm/utils/TableGen/Basic/CMakeLists.txt +++ b/llvm/utils/TableGen/Basic/CMakeLists.txt @@ -8,7 +8,7 @@ set(LLVM_LINK_COMPONENTS TableGen ) -add_llvm_library(LLVMTableGenBasic STATIC OBJECT EXCLUDE_FROM_ALL +add_llvm_library(LLVMTableGenBasic OBJECT EXCLUDE_FROM_ALL CodeGenIntrinsics.cpp SDNodeProperties.cpp ) diff --git a/llvm/utils/TableGen/CMakeLists.txt b/llvm/utils/TableGen/CMakeLists.txt index 14690329cabf..577aeded4be7 100644 --- a/llvm/utils/TableGen/CMakeLists.txt +++ b/llvm/utils/TableGen/CMakeLists.txt @@ -75,10 +75,10 @@ add_tablegen(llvm-tblgen LLVM X86MnemonicTables.cpp X86ModRMFilters.cpp X86RecognizableInstr.cpp + $ $ DEPENDS intrinsics_gen # via llvm-min-tablegen ) -target_link_libraries(llvm-tblgen PRIVATE LLVMTableGenCommon) set_target_properties(llvm-tblgen PROPERTIES FOLDER "Tablegenning") diff --git a/llvm/utils/TableGen/Common/CMakeLists.txt b/llvm/utils/TableGen/Common/CMakeLists.txt index 491d9bd2949d..0e985a083247 100644 --- a/llvm/utils/TableGen/Common/CMakeLists.txt +++ b/llvm/utils/TableGen/Common/CMakeLists.txt @@ -39,7 +39,6 @@ add_llvm_library(LLVMTableGenCommon STATIC OBJECT EXCLUDE_FROM_ALL vt_gen ) set_target_properties(LLVMTableGenCommon PROPERTIES FOLDER "Tablegenning") -target_link_libraries(LLVMTableGenCommon PUBLIC LLVMTableGenBasic) # Users may include its headers as "Common/*.h" target_include_directories(LLVMTableGenCommon -- GitLab From 336bdf1a255571f8d894e8befe4be7e9141f7541 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Storsj=C3=B6?= Date: Mon, 25 Mar 2024 11:52:29 +0200 Subject: [PATCH 200/306] [verify-uselistorder] Fix a typo, fix linking This fixes a typo from 2ef612050844355906e4b67d892a00bbb58c41d6, which broke the build with errors like: ld.lld: error: undefined symbol: LoadBitcodeIntoNewDbgInforFormat >>> referenced by verify-uselistorder.cpp >>> tools/verify-uselistorder/CMakeFiles/verify-uselistorder.dir/verify-uselistorder.cpp.o:(main) >>> did you mean: LoadBitcodeIntoNewDbgInfoFormat >>> defined in: lib/libLLVMBitReader.a(BitcodeReader.cpp.o) collect2: error: ld returned 1 exit status --- llvm/tools/verify-uselistorder/verify-uselistorder.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/llvm/tools/verify-uselistorder/verify-uselistorder.cpp b/llvm/tools/verify-uselistorder/verify-uselistorder.cpp index 8b299c394e4e..cb07dede1d13 100644 --- a/llvm/tools/verify-uselistorder/verify-uselistorder.cpp +++ b/llvm/tools/verify-uselistorder/verify-uselistorder.cpp @@ -68,7 +68,7 @@ static cl::opt cl::desc("Number of times to shuffle and verify use-lists"), cl::init(1), cl::cat(Cat)); -extern cl::opt LoadBitcodeIntoNewDbgInforFormat; +extern cl::opt LoadBitcodeIntoNewDbgInfoFormat; namespace { @@ -546,8 +546,8 @@ int main(int argc, char **argv) { "llvm tool to verify use-list order\n"); // Do not load bitcode into the new debug info format by default. - if (LoadBitcodeIntoNewDbgInforFormat == cl::boolOrDefault::BOU_UNSET) - LoadBitcodeIntoNewDbgInforFormat = cl::boolOrDefault::BOU_FALSE; + if (LoadBitcodeIntoNewDbgInfoFormat == cl::boolOrDefault::BOU_UNSET) + LoadBitcodeIntoNewDbgInfoFormat = cl::boolOrDefault::BOU_FALSE; LLVMContext Context; SMDiagnostic Err; -- GitLab From dbfc38ed6b3f2a9be0b1a86b2a074aad69eb58a6 Mon Sep 17 00:00:00 2001 From: Matthias Springer Date: Mon, 25 Mar 2024 18:57:53 +0900 Subject: [PATCH 201/306] [mlir][bufferization] Add `BufferOriginAnalysis` (#86461) This commit adds the `BufferOriginAnalysis`, which can be queried to check if two buffer SSA values originate from the same allocation. This new analysis is used in the buffer deallocation pass to fold away or simplify `bufferization.dealloc` ops more aggressively. The `BufferOriginAnalysis` is based on the `BufferViewFlowAnalysis`, which collects buffer SSA value "same buffer" dependencies. E.g., given IR such as: ``` %0 = memref.alloc() %1 = memref.subview %0 %2 = memref.subview %1 ``` The `BufferViewFlowAnalysis` will report the following "reverse" dependencies (`resolveReverse`) for `%2`: {`%2`, `%1`, `%0`}. I.e., all buffer SSA values in the reverse use-def chain that originate from the same allocation as `%2`. The `BufferOriginAnalysis` is built on top of that. It handles only simple cases at the moment and may conservatively return "unknown" around certain IR with branches, memref globals and function arguments. This analysis enables additional simplifications during `-buffer-deallocation-simplification`. In particular, "regular" scf.for loop nests, that yield buffers (or reallocations thereof) in the same order as they appear in the iter_args, are now handled much more efficiently. Such IR patterns are generated by the sparse compiler. --- .../Bufferization/IR/BufferizationOps.td | 1 + .../Transforms/BufferViewFlowAnalysis.h | 36 ++++ .../BufferDeallocationSimplification.cpp | 80 ++++----- .../Transforms/BufferViewFlowAnalysis.cpp | 160 ++++++++++++++++-- .../dealloc-loops.mlir | 86 ++++++++++ .../buffer-deallocation-simplification.mlir | 14 +- 6 files changed, 321 insertions(+), 56 deletions(-) create mode 100644 mlir/test/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation/dealloc-loops.mlir diff --git a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationOps.td b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationOps.td index 9dc6afcaab31..4f609ddff9a4 100644 --- a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationOps.td +++ b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationOps.td @@ -10,6 +10,7 @@ #define BUFFERIZATION_OPS include "mlir/Dialect/Bufferization/IR/AllocationOpInterface.td" +include "mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.td" include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.td" include "mlir/Dialect/Bufferization/IR/BufferizationBase.td" include "mlir/Interfaces/DestinationStyleOpInterface.td" diff --git a/mlir/include/mlir/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.h b/mlir/include/mlir/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.h index 9e43265c5dfe..4015231c845d 100644 --- a/mlir/include/mlir/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.h +++ b/mlir/include/mlir/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.h @@ -53,6 +53,7 @@ public: /// /// Results in resolve(B) returning {B, C} ValueSetT resolve(Value value) const; + ValueSetT resolveReverse(Value value) const; /// Removes the given values from all alias sets. void remove(const SetVector &aliasValues); @@ -73,11 +74,46 @@ private: /// Maps values to all immediate dependencies this value can have. ValueMapT dependencies; + ValueMapT reverseDependencies; /// A set of all SSA values that may be terminal buffers. DenseSet terminals; }; +/// An is-same-buffer analysis that checks if two SSA values belong to the same +/// buffer allocation or not. +class BufferOriginAnalysis { +public: + BufferOriginAnalysis(Operation *op); + + /// Return "true" if `v1` and `v2` originate from the same buffer allocation. + /// Return "false" if `v1` and `v2` originate from different allocations. + /// Return "nullopt" if we do not know for sure. + /// + /// Example 1: isSameAllocation(%0, %1) == true + /// ``` + /// %0 = memref.alloc() + /// %1 = memref.subview %0 + /// ``` + /// + /// Example 2: isSameAllocation(%0, %1) == false + /// ``` + /// %0 = memref.alloc() + /// %1 = memref.alloc() + /// ``` + /// + /// Example 3: isSameAllocation(%0, %2) == nullopt + /// ``` + /// %0 = memref.alloc() + /// %1 = memref.alloc() + /// %2 = arith.select %c, %0, %1 + /// ``` + std::optional isSameAllocation(Value v1, Value v2); + +private: + BufferViewFlowAnalysis analysis; +}; + } // namespace mlir #endif // MLIR_DIALECT_BUFFERIZATION_TRANSFORMS_BUFFERVIEWFLOWANALYSIS_H diff --git a/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp b/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp index e30779868b47..954485cfede3 100644 --- a/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp +++ b/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp @@ -12,8 +12,8 @@ // //===----------------------------------------------------------------------===// -#include "mlir/Analysis/AliasAnalysis.h" #include "mlir/Dialect/Bufferization/IR/Bufferization.h" +#include "mlir/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.h" #include "mlir/Dialect/Bufferization/Transforms/Passes.h" #include "mlir/Dialect/Func/IR/FuncOps.h" #include "mlir/Dialect/MemRef/IR/MemRef.h" @@ -34,6 +34,14 @@ using namespace mlir::bufferization; // Helpers //===----------------------------------------------------------------------===// +/// Given a memref value, return the "base" value by skipping over all +/// ViewLikeOpInterface ops (if any) in the reverse use-def chain. +static Value getViewBase(Value value) { + while (auto viewLikeOp = value.getDefiningOp()) + value = viewLikeOp.getViewSource(); + return value; +} + static LogicalResult updateDeallocIfChanged(DeallocOp deallocOp, ValueRange memrefs, ValueRange conditions, @@ -49,14 +57,6 @@ static LogicalResult updateDeallocIfChanged(DeallocOp deallocOp, return success(); } -/// Given a memref value, return the "base" value by skipping over all -/// ViewLikeOpInterface ops (if any) in the reverse use-def chain. -static Value getViewBase(Value value) { - while (auto viewLikeOp = value.getDefiningOp()) - value = viewLikeOp.getViewSource(); - return value; -} - /// Return "true" if the given values are guaranteed to be different (and /// non-aliasing) allocations based on the fact that one value is the result /// of an allocation and the other value is a block argument of a parent block. @@ -80,12 +80,14 @@ static bool distinctAllocAndBlockArgument(Value v1, Value v2) { /// Checks if `memref` may potentially alias a MemRef in `otherList`. It is /// often a requirement of optimization patterns that there cannot be any /// aliasing memref in order to perform the desired simplification. -static bool potentiallyAliasesMemref(AliasAnalysis &analysis, +static bool potentiallyAliasesMemref(BufferOriginAnalysis &analysis, ValueRange otherList, Value memref) { for (auto other : otherList) { if (distinctAllocAndBlockArgument(other, memref)) continue; - if (!analysis.alias(other, memref).isNo()) + std::optional analysisResult = + analysis.isSameAllocation(other, memref); + if (!analysisResult.has_value() || analysisResult == true) return true; } return false; @@ -129,8 +131,8 @@ namespace { struct RemoveDeallocMemrefsContainedInRetained : public OpRewritePattern { RemoveDeallocMemrefsContainedInRetained(MLIRContext *context, - AliasAnalysis &aliasAnalysis) - : OpRewritePattern(context), aliasAnalysis(aliasAnalysis) {} + BufferOriginAnalysis &analysis) + : OpRewritePattern(context), analysis(analysis) {} /// The passed 'memref' must not have a may-alias relation to any retained /// memref, and at least one must-alias relation. If there is no must-aliasing @@ -147,10 +149,11 @@ struct RemoveDeallocMemrefsContainedInRetained // deallocated in some situations and can thus not be dropped). bool atLeastOneMustAlias = false; for (Value retained : deallocOp.getRetained()) { - AliasResult analysisResult = aliasAnalysis.alias(retained, memref); - if (analysisResult.isMay()) + std::optional analysisResult = + analysis.isSameAllocation(retained, memref); + if (!analysisResult.has_value()) return failure(); - if (analysisResult.isMust() || analysisResult.isPartial()) + if (analysisResult == true) atLeastOneMustAlias = true; } if (!atLeastOneMustAlias) @@ -161,8 +164,9 @@ struct RemoveDeallocMemrefsContainedInRetained // we can remove that operand later on. for (auto [i, retained] : llvm::enumerate(deallocOp.getRetained())) { Value updatedCondition = deallocOp.getUpdatedConditions()[i]; - AliasResult analysisResult = aliasAnalysis.alias(retained, memref); - if (analysisResult.isMust() || analysisResult.isPartial()) { + std::optional analysisResult = + analysis.isSameAllocation(retained, memref); + if (analysisResult == true) { auto disjunction = rewriter.create( deallocOp.getLoc(), updatedCondition, cond); rewriter.replaceAllUsesExcept(updatedCondition, disjunction.getResult(), @@ -206,7 +210,7 @@ struct RemoveDeallocMemrefsContainedInRetained } private: - AliasAnalysis &aliasAnalysis; + BufferOriginAnalysis &analysis; }; /// Remove memrefs from the `retained` list which are guaranteed to not alias @@ -228,15 +232,15 @@ private: struct RemoveRetainedMemrefsGuaranteedToNotAlias : public OpRewritePattern { RemoveRetainedMemrefsGuaranteedToNotAlias(MLIRContext *context, - AliasAnalysis &aliasAnalysis) - : OpRewritePattern(context), aliasAnalysis(aliasAnalysis) {} + BufferOriginAnalysis &analysis) + : OpRewritePattern(context), analysis(analysis) {} LogicalResult matchAndRewrite(DeallocOp deallocOp, PatternRewriter &rewriter) const override { SmallVector newRetainedMemrefs, replacements; for (auto retainedMemref : deallocOp.getRetained()) { - if (potentiallyAliasesMemref(aliasAnalysis, deallocOp.getMemrefs(), + if (potentiallyAliasesMemref(analysis, deallocOp.getMemrefs(), retainedMemref)) { newRetainedMemrefs.push_back(retainedMemref); replacements.push_back({}); @@ -264,7 +268,7 @@ struct RemoveRetainedMemrefsGuaranteedToNotAlias } private: - AliasAnalysis &aliasAnalysis; + BufferOriginAnalysis &analysis; }; /// Split off memrefs to separate dealloc operations to reduce the number of @@ -297,8 +301,8 @@ private: struct SplitDeallocWhenNotAliasingAnyOther : public OpRewritePattern { SplitDeallocWhenNotAliasingAnyOther(MLIRContext *context, - AliasAnalysis &aliasAnalysis) - : OpRewritePattern(context), aliasAnalysis(aliasAnalysis) {} + BufferOriginAnalysis &analysis) + : OpRewritePattern(context), analysis(analysis) {} LogicalResult matchAndRewrite(DeallocOp deallocOp, PatternRewriter &rewriter) const override { @@ -314,7 +318,7 @@ struct SplitDeallocWhenNotAliasingAnyOther SmallVector otherMemrefs(deallocOp.getMemrefs()); otherMemrefs.erase(otherMemrefs.begin() + i); // Check if `memref` can split off into a separate bufferization.dealloc. - if (potentiallyAliasesMemref(aliasAnalysis, otherMemrefs, memref)) { + if (potentiallyAliasesMemref(analysis, otherMemrefs, memref)) { // `memref` alias with other memrefs, do not split off. remainingMemrefs.push_back(memref); remainingConditions.push_back(cond); @@ -352,7 +356,7 @@ struct SplitDeallocWhenNotAliasingAnyOther } private: - AliasAnalysis &aliasAnalysis; + BufferOriginAnalysis &analysis; }; /// Check for every retained memref if a must-aliasing memref exists in the @@ -381,8 +385,8 @@ private: struct RetainedMemrefAliasingAlwaysDeallocatedMemref : public OpRewritePattern { RetainedMemrefAliasingAlwaysDeallocatedMemref(MLIRContext *context, - AliasAnalysis &aliasAnalysis) - : OpRewritePattern(context), aliasAnalysis(aliasAnalysis) {} + BufferOriginAnalysis &analysis) + : OpRewritePattern(context), analysis(analysis) {} LogicalResult matchAndRewrite(DeallocOp deallocOp, PatternRewriter &rewriter) const override { @@ -396,8 +400,9 @@ struct RetainedMemrefAliasingAlwaysDeallocatedMemref if (!matchPattern(cond, m_One())) continue; - AliasResult analysisResult = aliasAnalysis.alias(retained, memref); - if (analysisResult.isMust() || analysisResult.isPartial()) { + std::optional analysisResult = + analysis.isSameAllocation(retained, memref); + if (analysisResult == true) { rewriter.replaceAllUsesWith(res, cond); aliasesWithConstTrueMemref[i] = true; canDropMemref = true; @@ -411,10 +416,9 @@ struct RetainedMemrefAliasingAlwaysDeallocatedMemref if (!extractOp) continue; - AliasResult extractAnalysisResult = - aliasAnalysis.alias(retained, extractOp.getOperand()); - if (extractAnalysisResult.isMust() || - extractAnalysisResult.isPartial()) { + std::optional extractAnalysisResult = + analysis.isSameAllocation(retained, extractOp.getOperand()); + if (extractAnalysisResult == true) { rewriter.replaceAllUsesWith(res, cond); aliasesWithConstTrueMemref[i] = true; canDropMemref = true; @@ -434,7 +438,7 @@ struct RetainedMemrefAliasingAlwaysDeallocatedMemref } private: - AliasAnalysis &aliasAnalysis; + BufferOriginAnalysis &analysis; }; } // namespace @@ -452,13 +456,13 @@ struct BufferDeallocationSimplificationPass : public bufferization::impl::BufferDeallocationSimplificationBase< BufferDeallocationSimplificationPass> { void runOnOperation() override { - AliasAnalysis &aliasAnalysis = getAnalysis(); + BufferOriginAnalysis analysis(getOperation()); RewritePatternSet patterns(&getContext()); patterns.add(&getContext(), - aliasAnalysis); + analysis); populateDeallocOpCanonicalizationPatterns(patterns, &getContext()); if (failed( diff --git a/mlir/lib/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.cpp b/mlir/lib/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.cpp index 9a36057425f3..72f47b8b468e 100644 --- a/mlir/lib/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.cpp +++ b/mlir/lib/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.cpp @@ -19,22 +19,23 @@ using namespace mlir; using namespace mlir::bufferization; +//===----------------------------------------------------------------------===// +// BufferViewFlowAnalysis +//===----------------------------------------------------------------------===// + /// Constructs a new alias analysis using the op provided. BufferViewFlowAnalysis::BufferViewFlowAnalysis(Operation *op) { build(op); } -/// Find all immediate and indirect dependent buffers this value could -/// potentially have. Note that the resulting set will also contain the value -/// provided as it is a dependent alias of itself. -BufferViewFlowAnalysis::ValueSetT -BufferViewFlowAnalysis::resolve(Value rootValue) const { - ValueSetT result; +static BufferViewFlowAnalysis::ValueSetT +resolveValues(const BufferViewFlowAnalysis::ValueMapT &map, Value value) { + BufferViewFlowAnalysis::ValueSetT result; SmallVector queue; - queue.push_back(rootValue); + queue.push_back(value); while (!queue.empty()) { Value currentValue = queue.pop_back_val(); if (result.insert(currentValue).second) { - auto it = dependencies.find(currentValue); - if (it != dependencies.end()) { + auto it = map.find(currentValue); + if (it != map.end()) { for (Value aliasValue : it->second) queue.push_back(aliasValue); } @@ -43,6 +44,19 @@ BufferViewFlowAnalysis::resolve(Value rootValue) const { return result; } +/// Find all immediate and indirect dependent buffers this value could +/// potentially have. Note that the resulting set will also contain the value +/// provided as it is a dependent alias of itself. +BufferViewFlowAnalysis::ValueSetT +BufferViewFlowAnalysis::resolve(Value rootValue) const { + return resolveValues(dependencies, rootValue); +} + +BufferViewFlowAnalysis::ValueSetT +BufferViewFlowAnalysis::resolveReverse(Value rootValue) const { + return resolveValues(reverseDependencies, rootValue); +} + /// Removes the given values from all alias sets. void BufferViewFlowAnalysis::remove(const SetVector &aliasValues) { for (auto &entry : dependencies) @@ -69,8 +83,10 @@ void BufferViewFlowAnalysis::rename(Value from, Value to) { void BufferViewFlowAnalysis::build(Operation *op) { // Registers all dependencies of the given values. auto registerDependencies = [&](ValueRange values, ValueRange dependencies) { - for (auto [value, dep] : llvm::zip_equal(values, dependencies)) + for (auto [value, dep] : llvm::zip_equal(values, dependencies)) { this->dependencies[value].insert(dep); + this->reverseDependencies[dep].insert(value); + } }; // Mark all buffer results and buffer region entry block arguments of the @@ -188,3 +204,127 @@ bool BufferViewFlowAnalysis::mayBeTerminalBuffer(Value value) const { assert(isa(value.getType()) && "expected memref"); return terminals.contains(value); } + +//===----------------------------------------------------------------------===// +// BufferOriginAnalysis +//===----------------------------------------------------------------------===// + +/// Return "true" if the given value is the result of a memory allocation. +static bool hasAllocateSideEffect(Value v) { + Operation *op = v.getDefiningOp(); + if (!op) + return false; + return hasEffect(op, v); +} + +/// Return "true" if the given value is a function block argument. +static bool isFunctionArgument(Value v) { + auto bbArg = dyn_cast(v); + if (!bbArg) + return false; + Block *b = bbArg.getOwner(); + auto funcOp = dyn_cast(b->getParentOp()); + if (!funcOp) + return false; + return bbArg.getOwner() == &funcOp.getFunctionBody().front(); +} + +/// Given a memref value, return the "base" value by skipping over all +/// ViewLikeOpInterface ops (if any) in the reverse use-def chain. +static Value getViewBase(Value value) { + while (auto viewLikeOp = value.getDefiningOp()) + value = viewLikeOp.getViewSource(); + return value; +} + +BufferOriginAnalysis::BufferOriginAnalysis(Operation *op) : analysis(op) {} + +std::optional BufferOriginAnalysis::isSameAllocation(Value v1, Value v2) { + assert(isa(v1.getType()) && "expected buffer"); + assert(isa(v2.getType()) && "expected buffer"); + + // Skip over all view-like ops. + v1 = getViewBase(v1); + v2 = getViewBase(v2); + + // Fast path: If both buffers are the same SSA value, we can be sure that + // they originate from the same allocation. + if (v1 == v2) + return true; + + // Compute the SSA values from which the buffers `v1` and `v2` originate. + SmallPtrSet origin1 = analysis.resolveReverse(v1); + SmallPtrSet origin2 = analysis.resolveReverse(v2); + + // Originating buffers are "terminal" if they could not be traced back any + // further by the `BufferViewFlowAnalysis`. Examples of terminal buffers: + // - function block arguments + // - values defined by allocation ops such as "memref.alloc" + // - values defined by ops that are unknown to the buffer view flow analysis + // - values that are marked as "terminal" in the `BufferViewFlowOpInterface` + SmallPtrSet terminal1, terminal2; + + // While gathering terminal buffers, keep track of whether all terminal + // buffers are newly allocated buffer or function entry arguments. + bool allAllocs1 = true, allAllocs2 = true; + bool allAllocsOrFuncEntryArgs1 = true, allAllocsOrFuncEntryArgs2 = true; + + // Helper function that gathers terminal buffers among `origin`. + auto gatherTerminalBuffers = [this](const SmallPtrSet &origin, + SmallPtrSet &terminal, + bool &allAllocs, + bool &allAllocsOrFuncEntryArgs) { + for (Value v : origin) { + if (isa(v.getType()) && analysis.mayBeTerminalBuffer(v)) { + terminal.insert(v); + allAllocs &= hasAllocateSideEffect(v); + allAllocsOrFuncEntryArgs &= + isFunctionArgument(v) || hasAllocateSideEffect(v); + } + } + assert(!terminal.empty() && "expected non-empty terminal set"); + }; + + // Gather terminal buffers for `v1` and `v2`. + gatherTerminalBuffers(origin1, terminal1, allAllocs1, + allAllocsOrFuncEntryArgs1); + gatherTerminalBuffers(origin2, terminal2, allAllocs2, + allAllocsOrFuncEntryArgs2); + + // If both `v1` and `v2` have a single matching terminal buffer, they are + // guaranteed to originate from the same buffer allocation. + if (llvm::hasSingleElement(terminal1) && llvm::hasSingleElement(terminal2) && + *terminal1.begin() == *terminal2.begin()) + return true; + + // At least one of the two values has multiple terminals. + + // Check if there is overlap between the terminal buffers of `v1` and `v2`. + bool distinctTerminalSets = true; + for (Value v : terminal1) + distinctTerminalSets &= !terminal2.contains(v); + // If there is overlap between the terminal buffers of `v1` and `v2`, we + // cannot make an accurate decision without further analysis. + if (!distinctTerminalSets) + return std::nullopt; + + // If `v1` originates from only allocs, and `v2` is guaranteed to originate + // from different allocations (that is guaranteed if `v2` originates from + // only distinct allocs or function entry arguments), we can be sure that + // `v1` and `v2` originate from different allocations. The same argument can + // be made when swapping `v1` and `v2`. + bool isolatedAlloc1 = allAllocs1 && (allAllocs2 || allAllocsOrFuncEntryArgs2); + bool isolatedAlloc2 = (allAllocs1 || allAllocsOrFuncEntryArgs1) && allAllocs2; + if (isolatedAlloc1 || isolatedAlloc2) + return false; + + // Otherwise: We do not know whether `v1` and `v2` originate from the same + // allocation or not. + // TODO: Function arguments are currently handled conservatively. We assume + // that they could be the same allocation. + // TODO: Terminals other than allocations and function arguments are + // currently handled conservatively. We assume that they could be the same + // allocation. E.g., we currently return "nullopt" for values that originate + // from different "memref.get_global" ops (with different symbols). + return std::nullopt; +} diff --git a/mlir/test/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation/dealloc-loops.mlir b/mlir/test/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation/dealloc-loops.mlir new file mode 100644 index 000000000000..53b28c3aab6f --- /dev/null +++ b/mlir/test/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation/dealloc-loops.mlir @@ -0,0 +1,86 @@ +// RUN: mlir-opt %s -expand-realloc="emit-deallocs=false" -ownership-based-buffer-deallocation="private-function-dynamic-ownership=true" -canonicalize -buffer-deallocation-simplification | FileCheck %s + +// A function that reallocates two buffer inside of a loop. The simplification +// pass should be able to figure out that the iter_args are always originating +// from different allocations. IR like this one appears in the sparse compiler. + +// CHECK-LABEL: func private @loop_with_realloc( +func.func private @loop_with_realloc(%lb: index, %ub: index, %step: index, %c: i1, %s1: index, %s2: index) -> (memref, memref) { + // CHECK-DAG: %[[false:.*]] = arith.constant false + // CHECK-DAG: %[[true:.*]] = arith.constant true + + // CHECK: %[[m0:.*]] = memref.alloc + %m0 = memref.alloc(%s1) : memref + // CHECK: %[[m1:.*]] = memref.alloc + %m1 = memref.alloc(%s1) : memref + + // CHECK: %[[r:.*]]:4 = scf.for {{.*}} iter_args(%[[arg0:.*]] = %[[m0]], %[[arg1:.*]] = %[[m1]], %[[o0:.*]] = %[[false]], %[[o1:.*]] = %[[false]]) + %r0, %r1 = scf.for %iv = %lb to %ub step %step iter_args(%arg0 = %m0, %arg1 = %m1) -> (memref, memref) { + // CHECK: %[[m2:.*]]:2 = scf.if %{{.*}} -> (memref, i1) { + // CHECK-NEXT: memref.alloc + // CHECK-NEXT: memref.subview + // CHECK-NEXT: memref.copy + // CHECK-NEXT: scf.yield %{{.*}}, %[[true]] + // CHECK-NEXT: } else { + // CHECK-NEXT: memref.reinterpret_cast + // CHECK-NEXT: scf.yield %{{.*}}, %[[false]] + // CHECK-NEXT: } + %m2 = memref.realloc %arg0(%s2) : memref to memref + // CHECK: %[[m3:.*]]:2 = scf.if %{{.*}} -> (memref, i1) { + // CHECK-NEXT: memref.alloc + // CHECK-NEXT: memref.subview + // CHECK-NEXT: memref.copy + // CHECK-NEXT: scf.yield %{{.*}}, %[[true]] + // CHECK-NEXT: } else { + // CHECK-NEXT: memref.reinterpret_cast + // CHECK-NEXT: scf.yield %{{.*}}, %[[false]] + // CHECK-NEXT: } + %m3 = memref.realloc %arg1(%s2) : memref to memref + + // CHECK: %[[base0:.*]], %{{.*}}, %{{.*}}, %{{.*}} = memref.extract_strided_metadata %[[arg0]] + // CHECK: %[[base1:.*]], %{{.*}}, %{{.*}}, %{{.*}} = memref.extract_strided_metadata %[[arg1]] + // CHECK: %[[d0:.*]] = bufferization.dealloc (%[[base0]] : memref) if (%[[o0]]) retain (%[[m2]]#0 : memref) + // CHECK: %[[d1:.*]] = bufferization.dealloc (%[[base1]] : memref) if (%[[o1]]) retain (%[[m3]]#0 : memref) + // CHECK-DAG: %[[o2:.*]] = arith.ori %[[d0]], %[[m2]]#1 + // CHECK-DAG: %[[o3:.*]] = arith.ori %[[d1]], %[[m3]]#1 + // CHECK: scf.yield %[[m2]]#0, %[[m3]]#0, %[[o2]], %[[o3]] + scf.yield %m2, %m3 : memref, memref + } + + // CHECK: %[[d2:.*]] = bufferization.dealloc (%[[m0]] : memref) if (%[[true]]) retain (%[[r]]#0 : memref) + // CHECK: %[[d3:.*]] = bufferization.dealloc (%[[m1]] : memref) if (%[[true]]) retain (%[[r]]#1 : memref) + // CHECK-DAG: %[[or0:.*]] = arith.ori %[[d2]], %[[r]]#2 + // CHECK-DAG: %[[or1:.*]] = arith.ori %[[d3]], %[[r]]#3 + // CHECK: return %[[r]]#0, %[[r]]#1, %[[or0]], %[[or1]] + return %r0, %r1 : memref, memref +} + +// ----- + +// The yielded values of the loop are swapped. Therefore, the +// bufferization.dealloc before the func.return can now longer be split, +// because %r0 could originate from either %m0 and %m1 (same for %r1). + +// CHECK-LABEL: func private @swapping_loop_with_realloc( +func.func private @swapping_loop_with_realloc(%lb: index, %ub: index, %step: index, %c: i1, %s1: index, %s2: index) -> (memref, memref) { + // CHECK-DAG: %[[false:.*]] = arith.constant false + // CHECK-DAG: %[[true:.*]] = arith.constant true + + // CHECK: %[[m0:.*]] = memref.alloc + %m0 = memref.alloc(%s1) : memref + // CHECK: %[[m1:.*]] = memref.alloc + %m1 = memref.alloc(%s1) : memref + + // CHECK: %[[r:.*]]:4 = scf.for {{.*}} iter_args(%[[arg0:.*]] = %[[m0]], %[[arg1:.*]] = %[[m1]], %[[o0:.*]] = %[[false]], %[[o1:.*]] = %[[false]]) + %r0, %r1 = scf.for %iv = %lb to %ub step %step iter_args(%arg0 = %m0, %arg1 = %m1) -> (memref, memref) { + %m2 = memref.realloc %arg0(%s2) : memref to memref + %m3 = memref.realloc %arg1(%s2) : memref to memref + scf.yield %m3, %m2 : memref, memref + } + + // CHECK: %[[base0:.*]], %{{.*}}, %{{.*}}, %{{.*}} = memref.extract_strided_metadata %[[r]]#0 + // CHECK: %[[base1:.*]], %{{.*}}, %{{.*}}, %{{.*}} = memref.extract_strided_metadata %[[r]]#1 + // CHECK: %[[d:.*]]:2 = bufferization.dealloc (%[[m0]], %[[m1]], %[[base0]], %[[base1]] : {{.*}}) if (%[[true]], %[[true]], %[[r]]#2, %[[r]]#3) retain (%[[r]]#0, %[[r]]#1 : {{.*}}) + // CHECK: return %[[r]]#0, %[[r]]#1, %[[d]]#0, %[[d]]#1 + return %r0, %r1 : memref, memref +} diff --git a/mlir/test/Dialect/Bufferization/Transforms/buffer-deallocation-simplification.mlir b/mlir/test/Dialect/Bufferization/Transforms/buffer-deallocation-simplification.mlir index eee69acbe821..b40a17cf800b 100644 --- a/mlir/test/Dialect/Bufferization/Transforms/buffer-deallocation-simplification.mlir +++ b/mlir/test/Dialect/Bufferization/Transforms/buffer-deallocation-simplification.mlir @@ -92,15 +92,13 @@ func.func @dealloc_split_when_no_other_aliasing(%arg0: i1, %arg1: memref<2xi32>, // CHECK-NEXT: [[ALLOC0:%.+]] = memref.alloc( // CHECK-NEXT: [[ALLOC1:%.+]] = memref.alloc( // CHECK-NEXT: [[V0:%.+]] = arith.select{{.*}}[[ALLOC0]], [[ALLOC1]] : -// COM: there is only one value in the retained list because the -// COM: RemoveRetainedMemrefsGuaranteedToNotAlias pattern also applies here and -// COM: removes %arg1 from the list. In the second dealloc, this does not apply -// COM: because function arguments are assumed potentially alias (even if the -// COM: types don't exactly match). +// COM: there is only one value in the retained lists because the +// COM: RemoveRetainedMemrefsGuaranteedToNotAlias pattern also applies here: +// COM: - %alloc is guaranteed to not alias with %arg1. +// COM: - %arg2 is guaranteed to not alias with %0. // CHECK-NEXT: [[V1:%.+]] = bufferization.dealloc ([[ALLOC0]] : memref<2xi32>) if ([[ARG0]]) retain ([[V0]] : memref<2xi32>) -// CHECK-NEXT: [[V2:%.+]]:2 = bufferization.dealloc ([[ARG2]] : memref<2xi32>) if ([[ARG3]]) retain ([[ARG1]], [[V0]] : memref<2xi32>, memref<2xi32>) -// CHECK-NEXT: [[V3:%.+]] = arith.ori [[V1]], [[V2]]#1 -// CHECK-NEXT: return [[V2]]#0, [[V3]] : +// CHECK-NEXT: [[V2:%.+]] = bufferization.dealloc ([[ARG2]] : memref<2xi32>) if ([[ARG3]]) retain ([[ARG1]] : memref<2xi32>) +// CHECK-NEXT: return [[V2]], [[V1]] : // ----- -- GitLab From 94a550dab26c4b30a187dd8e3ce431e0f915923b Mon Sep 17 00:00:00 2001 From: Mariusz Sikora Date: Mon, 25 Mar 2024 11:00:59 +0100 Subject: [PATCH 202/306] [AMDGPU][NFC] Rename Feature GFX11FullVGPRs to 1_5xVGPRs (#86468) --- llvm/lib/Target/AMDGPU/AMDGPU.td | 14 ++++++++------ llvm/lib/Target/AMDGPU/GCNSubtarget.h | 4 ++-- llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp | 4 ++-- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td index c877658cd38e..37dcfef3b2a3 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPU.td +++ b/llvm/lib/Target/AMDGPU/AMDGPU.td @@ -831,10 +831,12 @@ def FeatureNoDataDepHazard : SubtargetFeature<"no-data-dep-hazard", "Does not need SW waitstates" >; -def FeatureGFX11FullVGPRs : SubtargetFeature<"gfx11-full-vgprs", - "HasGFX11FullVGPRs", +// Allocate 1536 VGPRs for wave32 and 768 VGPRs for wave64 +// with allocation granularity 24 for wave32 and 12 for wave64 +def Feature1_5xVGPRs : SubtargetFeature<"allocate1_5xvgprs", + "Has1_5xVGPRs", "true", - "GFX11 with 50% more physical VGPRs and 50% larger allocation granule than GFX10" + "Has 50% more physical VGPRs and 50% larger allocation granule" >; @@ -1491,12 +1493,12 @@ def FeatureISAVersion11_0_Common : FeatureSet< def FeatureISAVersion11_0_0 : FeatureSet< !listconcat(FeatureISAVersion11_0_Common.Features, - [FeatureGFX11FullVGPRs, + [Feature1_5xVGPRs, FeatureUserSGPRInit16Bug])>; def FeatureISAVersion11_0_1 : FeatureSet< !listconcat(FeatureISAVersion11_0_Common.Features, - [FeatureGFX11FullVGPRs])>; + [Feature1_5xVGPRs])>; def FeatureISAVersion11_0_2 : FeatureSet< !listconcat(FeatureISAVersion11_0_Common.Features, @@ -1517,7 +1519,7 @@ def FeatureISAVersion11_5_1 : FeatureSet< [FeatureSALUFloatInsts, FeatureDPPSrc1SGPR, FeatureVGPRSingleUseHintInsts, - FeatureGFX11FullVGPRs])>; + Feature1_5xVGPRs])>; def FeatureISAVersion12 : FeatureSet< [FeatureGFX12, diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.h b/llvm/lib/Target/AMDGPU/GCNSubtarget.h index ca51da659c33..4da10beabe31 100644 --- a/llvm/lib/Target/AMDGPU/GCNSubtarget.h +++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.h @@ -223,7 +223,7 @@ protected: bool HasImageStoreD16Bug = false; bool HasImageGather4D16Bug = false; bool HasMSAALoadDstSelBug = false; - bool HasGFX11FullVGPRs = false; + bool Has1_5xVGPRs = false; bool HasMADIntraFwdBug = false; bool HasVOPDInsts = false; bool HasVALUTransUseHazard = false; @@ -1202,7 +1202,7 @@ public: /// target. bool hasNullExportTarget() const { return !GFX11Insts; } - bool hasGFX11FullVGPRs() const { return HasGFX11FullVGPRs; } + bool has1_5xVGPRs() const { return Has1_5xVGPRs; } bool hasVOPDInsts() const { return HasVOPDInsts; } diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp index a90dc32d396f..7bb84d78442b 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp @@ -1087,7 +1087,7 @@ unsigned getVGPRAllocGranule(const MCSubtargetInfo *STI, *EnableWavefrontSize32 : STI->getFeatureBits().test(FeatureWavefrontSize32); - if (STI->getFeatureBits().test(FeatureGFX11FullVGPRs)) + if (STI->getFeatureBits().test(Feature1_5xVGPRs)) return IsWave32 ? 24 : 12; if (hasGFX10_3Insts(*STI)) @@ -1114,7 +1114,7 @@ unsigned getTotalNumVGPRs(const MCSubtargetInfo *STI) { if (!isGFX10Plus(*STI)) return 256; bool IsWave32 = STI->getFeatureBits().test(FeatureWavefrontSize32); - if (STI->getFeatureBits().test(FeatureGFX11FullVGPRs)) + if (STI->getFeatureBits().test(Feature1_5xVGPRs)) return IsWave32 ? 1536 : 768; return IsWave32 ? 1024 : 512; } -- GitLab From 37785fedabd8fa752129ef5bac3462311af91c35 Mon Sep 17 00:00:00 2001 From: Discookie Date: Mon, 25 Mar 2024 10:08:56 +0000 Subject: [PATCH 203/306] [clang][analyzer] Bring cplusplus.ArrayDelete out of alpha (#83985) The checker finds a type of undefined behavior, where if the type of a pointer to an object-array is different from the objects' underlying type, calling `delete[]` is undefined, as the size of the two objects might be different. The checker has been in alpha for a while now, it is a simple checker that causes no crashes, and considering the severity of the issue, it has a low result-count on open-source projects (in my last test-run on my usual projects, it had 0 results). This commit cleans up the documentation and adds docs for the limitation related to tracking through references, in addition to moving it to `cplusplus`. --------- Co-authored-by: Balazs Benics Co-authored-by: whisperity --- clang/docs/analyzer/checkers.rst | 69 ++++++++++++------- .../clang/StaticAnalyzer/Checkers/Checkers.td | 10 +-- .../Checkers/CXXDeleteChecker.cpp | 4 +- clang/test/Analysis/ArrayDelete.cpp | 2 +- clang/www/analyzer/alpha_checks.html | 20 ------ clang/www/analyzer/available_checks.html | 27 ++++++++ 6 files changed, 80 insertions(+), 52 deletions(-) diff --git a/clang/docs/analyzer/checkers.rst b/clang/docs/analyzer/checkers.rst index fe2115149142..66da1c7b35f2 100644 --- a/clang/docs/analyzer/checkers.rst +++ b/clang/docs/analyzer/checkers.rst @@ -340,6 +340,51 @@ cplusplus C++ Checkers. +.. _cplusplus-ArrayDelete: + +cplusplus.ArrayDelete (C++) +""""""""""""""""""""""""""" + +Reports destructions of arrays of polymorphic objects that are destructed as +their base class. If the dynamic type of the array is different from its static +type, calling `delete[]` is undefined. + +This checker corresponds to the SEI CERT rule `EXP51-CPP: Do not delete an array through a pointer of the incorrect type `_. + +.. code-block:: cpp + + class Base { + public: + virtual ~Base() {} + }; + class Derived : public Base {}; + + Base *create() { + Base *x = new Derived[10]; // note: Casting from 'Derived' to 'Base' here + return x; + } + + void foo() { + Base *x = create(); + delete[] x; // warn: Deleting an array of 'Derived' objects as their base class 'Base' is undefined + } + +**Limitations** + +The checker does not emit note tags when casting to and from reference types, +even though the pointer values are tracked across references. + +.. code-block:: cpp + + void foo() { + Derived *d = new Derived[10]; + Derived &dref = *d; + + Base &bref = static_cast(dref); // no note + Base *b = &bref; + delete[] b; // warn: Deleting an array of 'Derived' objects as their base class 'Base' is undefined + } + .. _cplusplus-InnerPointer: cplusplus.InnerPointer (C++) @@ -2139,30 +2184,6 @@ Either the comparison is useless or there is division by zero. alpha.cplusplus ^^^^^^^^^^^^^^^ -.. _alpha-cplusplus-ArrayDelete: - -alpha.cplusplus.ArrayDelete (C++) -""""""""""""""""""""""""""""""""" -Reports destructions of arrays of polymorphic objects that are destructed as their base class. -This checker corresponds to the CERT rule `EXP51-CPP: Do not delete an array through a pointer of the incorrect type `_. - -.. code-block:: cpp - - class Base { - virtual ~Base() {} - }; - class Derived : public Base {} - - Base *create() { - Base *x = new Derived[10]; // note: Casting from 'Derived' to 'Base' here - return x; - } - - void foo() { - Base *x = create(); - delete[] x; // warn: Deleting an array of 'Derived' objects as their base class 'Base' is undefined - } - .. _alpha-cplusplus-DeleteWithNonVirtualDtor: alpha.cplusplus.DeleteWithNonVirtualDtor (C++) diff --git a/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td b/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td index 686e5e99f4a6..bf46766d44b3 100644 --- a/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td +++ b/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td @@ -622,6 +622,11 @@ def BlockInCriticalSectionChecker : Checker<"BlockInCriticalSection">, let ParentPackage = Cplusplus in { +def ArrayDeleteChecker : Checker<"ArrayDelete">, + HelpText<"Reports destructions of arrays of polymorphic objects that are " + "destructed as their base class.">, + Documentation; + def InnerPointerChecker : Checker<"InnerPointer">, HelpText<"Check for inner pointers of C++ containers used after " "re/deallocation">, @@ -777,11 +782,6 @@ def ContainerModeling : Checker<"ContainerModeling">, Documentation, Hidden; -def CXXArrayDeleteChecker : Checker<"ArrayDelete">, - HelpText<"Reports destructions of arrays of polymorphic objects that are " - "destructed as their base class.">, - Documentation; - def DeleteWithNonVirtualDtorChecker : Checker<"DeleteWithNonVirtualDtor">, HelpText<"Reports destructions of polymorphic objects with a non-virtual " "destructor in their base class">, diff --git a/clang/lib/StaticAnalyzer/Checkers/CXXDeleteChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/CXXDeleteChecker.cpp index b4dee1e300e8..1b1226a7f1a7 100644 --- a/clang/lib/StaticAnalyzer/Checkers/CXXDeleteChecker.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/CXXDeleteChecker.cpp @@ -220,11 +220,11 @@ CXXDeleteChecker::PtrCastVisitor::VisitNode(const ExplodedNode *N, /*addPosRange=*/true); } -void ento::registerCXXArrayDeleteChecker(CheckerManager &mgr) { +void ento::registerArrayDeleteChecker(CheckerManager &mgr) { mgr.registerChecker(); } -bool ento::shouldRegisterCXXArrayDeleteChecker(const CheckerManager &mgr) { +bool ento::shouldRegisterArrayDeleteChecker(const CheckerManager &mgr) { return true; } diff --git a/clang/test/Analysis/ArrayDelete.cpp b/clang/test/Analysis/ArrayDelete.cpp index 3b8d49552376..6887e0a35fb8 100644 --- a/clang/test/Analysis/ArrayDelete.cpp +++ b/clang/test/Analysis/ArrayDelete.cpp @@ -1,4 +1,4 @@ -// RUN: %clang_cc1 -analyze -analyzer-checker=alpha.cplusplus.ArrayDelete -std=c++11 -verify -analyzer-output=text %s +// RUN: %clang_cc1 -analyze -analyzer-checker=cplusplus.ArrayDelete -std=c++11 -verify -analyzer-output=text %s struct Base { virtual ~Base() = default; diff --git a/clang/www/analyzer/alpha_checks.html b/clang/www/analyzer/alpha_checks.html index 7bbe4a20288f..f040d1957b0f 100644 --- a/clang/www/analyzer/alpha_checks.html +++ b/clang/www/analyzer/alpha_checks.html @@ -307,26 +307,6 @@ void test(int x) { - - - -