From 36fd0e797974b75623d847e30d8ba0494e43af99 Mon Sep 17 00:00:00 2001 From: Kyungwoo Lee Date: Fri, 15 Mar 2024 04:55:12 -0700 Subject: [PATCH 001/514] Revert "[llvm-objdump][macho] Add support for ObjC relative method lists (#84250)" This reverts llvm/llvm-project#84250, commit aa6100643c2c8f9a1b06ba557b68b0fba477e3c7. See build failures: https://lab.llvm.org/buildbot/#/builders/178/builds/7028 https://lab.llvm.org/buildbot/#/builders/182/builds/9282 https://lab.llvm.org/buildbot/#/builders/186/builds/15299 https://lab.llvm.org/buildbot/#/builders/187/builds/14564 --- .../Inputs/rel-method-lists-arm64.dylib | Bin 3160 -> 0 bytes .../Inputs/rel-method-lists-arm64_32.dylib | Bin 2716 -> 0 bytes .../AArch64/macho-relative-method-lists.test | 87 -------------- llvm/tools/llvm-objdump/MachODump.cpp | 112 +----------------- 4 files changed, 2 insertions(+), 197 deletions(-) delete mode 100755 llvm/test/tools/llvm-objdump/MachO/AArch64/Inputs/rel-method-lists-arm64.dylib delete mode 100755 llvm/test/tools/llvm-objdump/MachO/AArch64/Inputs/rel-method-lists-arm64_32.dylib delete mode 100644 llvm/test/tools/llvm-objdump/MachO/AArch64/macho-relative-method-lists.test diff --git a/llvm/test/tools/llvm-objdump/MachO/AArch64/Inputs/rel-method-lists-arm64.dylib b/llvm/test/tools/llvm-objdump/MachO/AArch64/Inputs/rel-method-lists-arm64.dylib deleted file mode 100755 index 9b39fc98fb5d96b544894f21e1f7857891735e6f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3160 zcmcguO=whC6h6L*P1>X}!3dT{h*m`H&%7BVgwjH$GXs;ViJFL!BDXJZp3TU6^M=mT zByuiyk<0?|1IK z-@EsmbI+Y$pFjDh3t$tVyn)({y0jVKd*~q(yo0)eqe@NWCnrKv!;jhC06{QPfGEvS zDsX0l`dppw8BIo+jT#_qa6fF?U_K6&@~6(&%66@4wQPBg&gar$P&%Rp1T$>pMkDii zPB2Y$P3H}mufzm`UxP20t2Fc7*Y+p4HQ(9tOsQN{rSfS%+;3w%9x-2m2?#g%Pl&mt zxIRKvDwmnasO;G2co&#C1U)xqWsRFN(G*hZ-c@@`Dc$!Mdb&~$s zKuliBvux!S2M>HyoO4T4$Oy0tVBW-*_CeohB`Y{y!9mIuns?;QsMeWpApvfTiN zhj9V-ORnF?nD$Sy1C{14A=Nc9yoS8U!tKA>VQn=5YuLZIoKScF58L6nER|~ri?*Ut z>lc-2YB;6gG?ehpS!LUaEom~PnrEfqTxxLG{CJ35LjLY&-^5$W2YPD`U`y(89i=|S zqzhN}aI~;v$i2iN7)GTS(>!?(q>h`+^BuQh&oREF*(Vtj*Irg^Y9jBK)T!D)i;nE; zs`eduXK!lu4#q!g_I}3Xr}aekwZipuy*Y8)0wr-wZZVcRvX`;U6MyUTF7Wqycf!$K z9VfHL@~F8=$E0v?QuzPF82jw=tb+dN&w}Gsf;k1FU}O(v#>dreB^jAJoS(?(#*qX( z?ZMzK^x`c&7g$p+KI67!PdiXdz;Z7R-NK@sXlGN0x15@GnV)a zfC_!8J1erT=SF_%BIif$YcR2AKm3A&G0=3k%~P$n8pZSbKq$;`>fgCSCRT!C*L&Xgu?5+i6g(=q;mFWX6J(#9C)8uRr=P1PG`d>AEb z$s)iw^pvKb8E<_(%f)JOeTHjz14GK6A=yuIoQ~N?E{0NR|19k3pZNA5)FJ5@bc_Kce zaJ&Jh=ZLWkNjuYzXZ$pnb(;FzobNd~%XJoJ{j;{!$f>|<=JEyAvb?72Fym(3b#nf> zAXd#=fj#H9lw133^gylc)@Kp(eUZgHB){Kn8pbI?o-UL~G0Y{vxF2}uUpYXD!oneXURKKzDg1qaMCVb9gHjc{gx2wx=Zv!s_H-WDK-vx${*uD(?WsHZQKL~sQcrWlMa2t3G_(;3Nt6IVfJG}b+n#$)B z<3eIQB6ZvcR>O7-mz1iIwi=Q1V~6sekA`Pp7&(wk$=$#=Y7~-O&=(>xd$1jz(JP6; zV027*=1GUZwBf!Xllac?y-E_lV-oF7zr=18x<}xrV7@m?;9KCw`kTBnNIyVQ$6R6^ z??lqOyQMiaJw9DNR%~Xg#jmAD;qC67`r^xylHurrb-ZTKRx&BQrNhOU8MQ~nD^osF zsTPyUiHxicpxDO)ILfwV&*6nTA6T=lBeje?=_zAFY5yqrh8e8Gf2j)e A#{d8T diff --git a/llvm/test/tools/llvm-objdump/MachO/AArch64/macho-relative-method-lists.test b/llvm/test/tools/llvm-objdump/MachO/AArch64/macho-relative-method-lists.test deleted file mode 100644 index be8b7f8f3b2c..000000000000 --- a/llvm/test/tools/llvm-objdump/MachO/AArch64/macho-relative-method-lists.test +++ /dev/null @@ -1,87 +0,0 @@ -RUN: llvm-objdump --macho --objc-meta-data %p/Inputs/rel-method-lists-arm64_32.dylib | FileCheck %s --check-prefix=CHK32 -RUN: llvm-otool -ov %p/Inputs/rel-method-lists-arm64_32.dylib | FileCheck %s --check-prefix=CHK32 - -RUN: llvm-objdump --macho --objc-meta-data %p/Inputs/rel-method-lists-arm64.dylib | FileCheck %s --check-prefix=CHK64 -RUN: llvm-otool -ov %p/Inputs/rel-method-lists-arm64.dylib | FileCheck %s --check-prefix=CHK64 - -CHK32: baseMethods 0x660 (struct method_list_t *) -CHK32-NEXT: entsize 12 (relative) -CHK32-NEXT: count 3 -CHK32-NEXT: name 0x144 (0x7ac) instance_method_00 -CHK32-NEXT: types 0x91 (0x6fd) v8@0:4 -CHK32-NEXT: imp 0xffffff18 (0x588) -[MyClass instance_method_00] -CHK32-NEXT: name 0x13c (0x7b0) instance_method_01 -CHK32-NEXT: types 0x85 (0x6fd) v8@0:4 -CHK32-NEXT: imp 0xffffff28 (0x5a4) -[MyClass instance_method_01] -CHK32-NEXT: name 0x134 (0x7b4) instance_method_02 -CHK32-NEXT: types 0x79 (0x6fd) v8@0:4 -CHK32-NEXT: imp 0xffffff38 (0x5c0) -[MyClass instance_method_02] - -CHK32: baseMethods 0x630 (struct method_list_t *) -CHK32-NEXT: entsize 12 (relative) -CHK32-NEXT: count 3 -CHK32-NEXT: name 0x180 (0x7b8) class_method_00 -CHK32-NEXT: types 0xc1 (0x6fd) v8@0:4 -CHK32-NEXT: imp 0xffffff9c (0x5dc) +[MyClass class_method_00] -CHK32-NEXT: name 0x178 (0x7bc) class_method_01 -CHK32-NEXT: types 0xb5 (0x6fd) v8@0:4 -CHK32-NEXT: imp 0xffffffac (0x5f8) +[MyClass class_method_01] -CHK32-NEXT: name 0x170 (0x7c0) class_method_02 -CHK32-NEXT: types 0xa9 (0x6fd) v8@0:4 -CHK32-NEXT: imp 0xffffffbc (0x614) +[MyClass class_method_02] - -CHK64: baseMethods 0x7d8 (struct method_list_t *) -CHK64-NEXT: entsize 24 -CHK64-NEXT: count 3 -CHK64-NEXT: name 0x6a4 instance_method_00 -CHK64-NEXT: types 0x6dd v16@0:8 -CHK64-NEXT: imp -[MyClass instance_method_00] -CHK64-NEXT: name 0x6b7 instance_method_01 -CHK64-NEXT: types 0x6dd v16@0:8 -CHK64-NEXT: imp -[MyClass instance_method_01] -CHK64-NEXT: name 0x6ca instance_method_02 -CHK64-NEXT: types 0x6dd v16@0:8 -CHK64-NEXT: imp -[MyClass instance_method_02] - -CHK64: baseMethods 0x740 (struct method_list_t *) -CHK64-NEXT: entsize 24 -CHK64-NEXT: count 3 -CHK64-NEXT: name 0x674 class_method_00 -CHK64-NEXT: types 0x6dd v16@0:8 -CHK64-NEXT: imp +[MyClass class_method_00] -CHK64-NEXT: name 0x684 class_method_01 -CHK64-NEXT: types 0x6dd v16@0:8 -CHK64-NEXT: imp +[MyClass class_method_01] -CHK64-NEXT: name 0x694 class_method_02 -CHK64-NEXT: types 0x6dd v16@0:8 -CHK64-NEXT: imp +[MyClass class_method_02] - - -######## Generate rel-method-lists-arm64.dylib ######## -// clang -c main.mm -o main.o -target arm64-apple-macos -arch arm64 -// ld64.ld64 -dylib -demangle -dynamic main.o -o rel-method-lists-arm64.dylib -syslibroot MacOSX14.2.sdk -segalign 0x10 -objc_relative_method_lists - -######## Generate rel-method-lists-arm64_32.dylib ######## -// clang -c main.mm -o main.o -target arm64_32-apple-watchos -arch arm64_32 -// ld64.ld64 -dylib -demangle -dynamic main.o -o rel-method-lists-arm64_32.dylib -syslibroot WatchOS.sdk -segalign 0x10 -objc_relative_method_lists - -// ~~~~~~~~~~~~~~~~~~~~~~~~~ main.mm ~~~~~~~~~~~~~~~~~~~~~~~~~ -__attribute__((objc_root_class)) -@interface MyClass -- (void)instance_method_00; -- (void)instance_method_01; -- (void)instance_method_02; -+ (void)class_method_00; -+ (void)class_method_01; -+ (void)class_method_02; -@end -@implementation MyClass -- (void)instance_method_00 {} -- (void)instance_method_01 {} -- (void)instance_method_02 {} -+ (void)class_method_00 {} -+ (void)class_method_01 {} -+ (void)class_method_02 {} -@end -void *_objc_empty_cache; -void *_objc_empty_vtable; diff --git a/llvm/tools/llvm-objdump/MachODump.cpp b/llvm/tools/llvm-objdump/MachODump.cpp index 1b0e5ba279d0..0e6935c0ac58 100644 --- a/llvm/tools/llvm-objdump/MachODump.cpp +++ b/llvm/tools/llvm-objdump/MachODump.cpp @@ -3661,10 +3661,6 @@ struct class_ro32_t { #define RO_ROOT (1 << 1) #define RO_HAS_CXX_STRUCTORS (1 << 2) -/* Values for method_list{64,32}_t->entsize */ -#define ML_HAS_RELATIVE_PTRS (1 << 31) -#define ML_ENTSIZE_MASK 0xFFFF - struct method_list64_t { uint32_t entsize; uint32_t count; @@ -3689,12 +3685,6 @@ struct method32_t { uint32_t imp; /* IMP (32-bit pointer) */ }; -struct method_relative_t { - int32_t name; /* SEL (32-bit relative) */ - int32_t types; /* const char * (32-bit relative) */ - int32_t imp; /* IMP (32-bit relative) */ -}; - struct protocol_list64_t { uint64_t count; /* uintptr_t (a 64-bit value) */ /* struct protocol64_t * list[0]; These pointers follow inline */ @@ -3996,12 +3986,6 @@ inline void swapStruct(struct method32_t &m) { sys::swapByteOrder(m.imp); } -inline void swapStruct(struct method_relative_t &m) { - sys::swapByteOrder(m.name); - sys::swapByteOrder(m.types); - sys::swapByteOrder(m.imp); -} - inline void swapStruct(struct protocol_list64_t &pl) { sys::swapByteOrder(pl.count); } @@ -4456,84 +4440,6 @@ static void print_layout_map32(uint32_t p, struct DisassembleInfo *info) { print_layout_map(layout_map, left); } -static void print_relative_method_list(uint32_t structSizeAndFlags, - uint32_t structCount, uint64_t p, - struct DisassembleInfo *info, - const char *indent, - uint32_t pointerBits) { - struct method_relative_t m; - const char *r, *name; - uint32_t offset, xoffset, left, i; - SectionRef S, xS; - - assert(((structSizeAndFlags & ML_HAS_RELATIVE_PTRS) != 0) && - "expected structSizeAndFlags to have ML_HAS_RELATIVE_PTRS flag"); - - outs() << indent << "\t\t entsize " - << (structSizeAndFlags & ML_ENTSIZE_MASK) << " (relative) \n"; - outs() << indent << "\t\t count " << structCount << "\n"; - - for (i = 0; i < structCount; i++) { - r = get_pointer_64(p, offset, left, S, info); - memset(&m, '\0', sizeof(struct method_relative_t)); - if (left < sizeof(struct method_relative_t)) { - memcpy(&m, r, left); - outs() << indent << " (method_t extends past the end of the section)\n"; - } else - memcpy(&m, r, sizeof(struct method_relative_t)); - if (info->O->isLittleEndian() != sys::IsLittleEndianHost) - swapStruct(m); - - outs() << indent << "\t\t name " << format("0x%" PRIx32, m.name); - uint64_t relNameRefVA = p + offsetof(struct method_relative_t, name); - uint64_t absNameRefVA = relNameRefVA + m.name; - outs() << " (" << format("0x%" PRIx32, absNameRefVA) << ")"; - - // since this is a relative list, absNameRefVA is the address of the - // __objc_selrefs entry, so a pointer, not the actual name - const char *nameRefPtr = - get_pointer_64(absNameRefVA, xoffset, left, xS, info); - if (nameRefPtr) { - uint32_t pointerSize = pointerBits / CHAR_BIT; - if (left < pointerSize) - outs() << indent << " (nameRefPtr extends past the end of the section)"; - else { - if (pointerSize == 64) { - name = get_pointer_64(*reinterpret_cast(nameRefPtr), - xoffset, left, xS, info); - } else { - name = get_pointer_32(*reinterpret_cast(nameRefPtr), - xoffset, left, xS, info); - } - if (name != nullptr) - outs() << format(" %.*s", left, name); - } - } - outs() << "\n"; - - outs() << indent << "\t\t types " << format("0x%" PRIx32, m.types); - uint64_t relTypesVA = p + offsetof(struct method_relative_t, types); - uint64_t absTypesVA = relTypesVA + m.types; - outs() << " (" << format("0x%" PRIx32, absTypesVA) << ")"; - name = get_pointer_32(absTypesVA, xoffset, left, xS, info); - if (name != nullptr) - outs() << format(" %.*s", left, name); - outs() << "\n"; - - outs() << indent << "\t\t imp " << format("0x%" PRIx32, m.imp); - uint64_t relImpVA = p + offsetof(struct method_relative_t, imp); - uint64_t absImpVA = relImpVA + m.imp; - outs() << " (" << format("0x%" PRIx32, absImpVA) << ")"; - name = GuessSymbolName(absImpVA, info->AddrMap); - if (name != nullptr) - outs() << " " << name; - outs() << "\n"; - - p += sizeof(struct method_relative_t); - offset += sizeof(struct method_relative_t); - } -} - static void print_method_list64_t(uint64_t p, struct DisassembleInfo *info, const char *indent) { struct method_list64_t ml; @@ -4555,17 +4461,10 @@ static void print_method_list64_t(uint64_t p, struct DisassembleInfo *info, memcpy(&ml, r, sizeof(struct method_list64_t)); if (info->O->isLittleEndian() != sys::IsLittleEndianHost) swapStruct(ml); - p += sizeof(struct method_list64_t); - - if ((ml.entsize & ML_HAS_RELATIVE_PTRS) != 0) { - print_relative_method_list(ml.entsize, ml.count, p, info, indent, - /*pointerBits=*/64); - return; - } - outs() << indent << "\t\t entsize " << ml.entsize << "\n"; outs() << indent << "\t\t count " << ml.count << "\n"; + p += sizeof(struct method_list64_t); offset += sizeof(struct method_list64_t); for (i = 0; i < ml.count; i++) { r = get_pointer_64(p, offset, left, S, info); @@ -4653,17 +4552,10 @@ static void print_method_list32_t(uint64_t p, struct DisassembleInfo *info, memcpy(&ml, r, sizeof(struct method_list32_t)); if (info->O->isLittleEndian() != sys::IsLittleEndianHost) swapStruct(ml); - p += sizeof(struct method_list32_t); - - if ((ml.entsize & ML_HAS_RELATIVE_PTRS) != 0) { - print_relative_method_list(ml.entsize, ml.count, p, info, indent, - /*pointerBits=*/32); - return; - } - outs() << indent << "\t\t entsize " << ml.entsize << "\n"; outs() << indent << "\t\t count " << ml.count << "\n"; + p += sizeof(struct method_list32_t); offset += sizeof(struct method_list32_t); for (i = 0; i < ml.count; i++) { r = get_pointer_32(p, offset, left, S, info); -- GitLab From 2c703ed7281cb0fb749bff75b1c313a0332bd9eb Mon Sep 17 00:00:00 2001 From: Eric Date: Fri, 15 Mar 2024 08:23:07 -0400 Subject: [PATCH 002/514] Rework Modules CMake to be (more) idiomatic. (#84936) - Fix bug in documentation regarding dependencies. - Rework Modules CMake to be (more) idiomatic. --- libcxx/docs/Modules.rst | 29 ++--------------------------- libcxx/modules/CMakeLists.txt.in | 16 ++++++++++++---- 2 files changed, 14 insertions(+), 31 deletions(-) diff --git a/libcxx/docs/Modules.rst b/libcxx/docs/Modules.rst index ee2b81d3b9e7..5b027ed1bd07 100644 --- a/libcxx/docs/Modules.rst +++ b/libcxx/docs/Modules.rst @@ -178,34 +178,13 @@ This is a small sample program that uses the module ``std``. It consists of a ) FetchContent_MakeAvailable(std) - # - # Adjust project compiler flags - # - - add_compile_options($<$:-fprebuilt-module-path=${std_BINARY_DIR}/CMakeFiles/std.dir/>) - add_compile_options($<$:-fprebuilt-module-path=${std_BINARY_DIR}/CMakeFiles/std.compat.dir/>) - add_compile_options($<$:-nostdinc++>) - # The include path needs to be set to be able to use macros from headers. - # For example from, the headers and . - add_compile_options($<$:-isystem>) - add_compile_options($<$:${LIBCXX_BUILD}/include/c++/v1>) - - # - # Adjust project linker flags - # - - add_link_options($<$:-nostdlib++>) - add_link_options($<$:-L${LIBCXX_BUILD}/lib>) - add_link_options($<$:-Wl,-rpath,${LIBCXX_BUILD}/lib>) - # Linking against the standard c++ library is required for CMake to get the proper dependencies. - link_libraries(std c++) - link_libraries(std.compat c++) - # # Add the project # add_executable(main) + add_dependencies(main std.compat) + target_link_libraries(main std.compat) target_sources(main PRIVATE main.cpp @@ -218,13 +197,9 @@ Building this project is done with the following steps, assuming the files $ mkdir build $ cmake -G Ninja -S . -B build -DCMAKE_CXX_COMPILER= -DLIBCXX_BUILD= - $ ninja -j1 std -C build $ ninja -C build $ build/main -.. note:: The ``std`` dependencies of ``std.compat`` is not always resolved when - building the ``std`` target using multiple jobs. - .. warning:: ```` should point point to the real binary and not to a symlink. diff --git a/libcxx/modules/CMakeLists.txt.in b/libcxx/modules/CMakeLists.txt.in index e332d70cc163..c35f6fecb1fd 100644 --- a/libcxx/modules/CMakeLists.txt.in +++ b/libcxx/modules/CMakeLists.txt.in @@ -50,10 +50,15 @@ endif() target_compile_options(std PUBLIC -nostdinc++ + @LIBCXX_COMPILE_FLAGS@ +) +target_compile_options(std + PRIVATE -Wno-reserved-module-identifier -Wno-reserved-user-defined-literal - @LIBCXX_COMPILE_FLAGS@ ) +target_link_options(std PUBLIC -nostdlib++ -Wl,-rpath,@LIBCXX_LIBRARY_DIR@ -L@LIBCXX_LIBRARY_DIR@) +target_link_libraries(std c++) set_target_properties(std PROPERTIES OUTPUT_NAME "c++std" @@ -67,7 +72,7 @@ target_sources(std.compat std.compat.cppm ) -target_include_directories(std.compat SYSTEM PRIVATE @LIBCXX_CONFIGURED_INCLUDE_DIRS@) +target_include_directories(std.compat SYSTEM PUBLIC @LIBCXX_CONFIGURED_INCLUDE_DIRS@) if (NOT @LIBCXX_ENABLE_EXCEPTIONS@) target_compile_options(std.compat PUBLIC -fno-exceptions) @@ -76,13 +81,16 @@ endif() target_compile_options(std.compat PUBLIC -nostdinc++ + @LIBCXX_COMPILE_FLAGS@ +) +target_compile_options(std.compat + PRIVATE -Wno-reserved-module-identifier -Wno-reserved-user-defined-literal - -fmodule-file=std=${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/std.dir/std.pcm - @LIBCXX_COMPILE_FLAGS@ ) set_target_properties(std.compat PROPERTIES OUTPUT_NAME "c++std.compat" ) add_dependencies(std.compat std) +target_link_libraries(std.compat PUBLIC std c++) -- GitLab From 5b5525d403f4b1e155c5fc50649b6c6d9e7d4de5 Mon Sep 17 00:00:00 2001 From: Jie Fu Date: Fri, 15 Mar 2024 20:30:05 +0800 Subject: [PATCH 003/514] [flang][OpenMP] Remove unused variable (NFC) llvm-project/flang/lib/Lower/OpenMP/ClauseProcessor.cpp:97:15: error: unused variable 'allocatorOperand' [-Werror,-Wunused-variable] mlir::Value allocatorOperand; ^ 1 error generated. --- flang/lib/Lower/OpenMP/ClauseProcessor.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp index 6e45a939333d..6fefd04f409b 100644 --- a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp +++ b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp @@ -94,7 +94,6 @@ genAllocateClause(Fortran::lower::AbstractConverter &converter, mlir::Location currentLocation = converter.getCurrentLocation(); Fortran::lower::StatementContext stmtCtx; - mlir::Value allocatorOperand; const omp::ObjectList &objectList = std::get(clause.t); const auto &modifier = std::get>(clause.t); -- GitLab From 435d4c12de6fdc8e67ceffa04f4d9fba9f006b2d Mon Sep 17 00:00:00 2001 From: Orlando Cazalet-Hyams Date: Fri, 15 Mar 2024 10:47:48 +0000 Subject: [PATCH 004/514] Reapply [RemoveDIs] Read/write DbgRecords directly from/to bitcode (#83251) Reaplying after revert in #85382 (861ebe6446296c96578807363aa292c69d827773). Fixed intermittent test failure by avoiding piping output in some RUN lines. If --write-experimental-debuginfo-iterators-to-bitcode is true (default false) and --expermental-debuginfo-iterators is also true then the new debug info format (non-instruction records) is written to bitcode directly. Added the following records: FUNC_CODE_DEBUG_RECORD_LABEL FUNC_CODE_DEBUG_RECORD_VALUE FUNC_CODE_DEBUG_RECORD_DECLARE FUNC_CODE_DEBUG_RECORD_ASSIGN FUNC_CODE_DEBUG_RECORD_VALUE_SIMPLE The last one has an abbrev in FUNCTION_BLOCK BLOCK_INFO. Incidentally, this uses the last value available without widening the code-length for FUNCTION_BLOCK from 4 to 5 bits. Records are formatted as follows: All DbgRecord start with: 1. DILocation FUNC_CODE_DEBUG_RECORD_LABEL 2. DILabel DPValues then share common fields: 2. DILocalVariable 3. DIExpression FUNC_CODE_DEBUG_RECORD_VALUE 4. Location Metadata FUNC_CODE_DEBUG_RECORD_DECLARE 4. Location Metadata FUNC_CODE_DEBUG_RECORD_VALUE_SIMPLE 4. Location Value (single) FUNC_CODE_DEBUG_RECORD_ASSIGN 4. Location Metadata 5. DIAssignID 6. DIExpression (address) 7. Location Metadata (address) Encoding the DILocation metadata reference directly appeared to yield smaller bitcode files than encoding the operands seperately (as is done with instruction DILocations). FUNC_CODE_DEBUG_RECORD_VALUE_SIMPLE is by far the most common DbgRecord record in optimized code (order of 5x-10x over other kinds). Unoptimized code should only contain FUNC_CODE_DEBUG_RECORD_DECLARE. --- llvm/include/llvm/Bitcode/LLVMBitCodes.h | 11 ++ llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp | 5 + llvm/lib/Bitcode/Reader/BitcodeReader.cpp | 101 +++++++++++- llvm/lib/Bitcode/Writer/BitcodeWriter.cpp | 120 +++++++++++--- llvm/lib/Bitcode/Writer/BitcodeWriterPass.cpp | 20 +-- llvm/lib/Bitcode/Writer/ValueEnumerator.cpp | 149 ++++++++++++------ llvm/lib/IR/BasicBlock.cpp | 5 + llvm/lib/Linker/IRMover.cpp | 21 ++- .../Transforms/IPO/ThinLTOBitcodeWriter.cpp | 9 +- llvm/test/Bitcode/dbg-record-roundtrip.ll | 131 +++++++++++++++ llvm/tools/llvm-as/llvm-as.cpp | 7 + .../verify-uselistorder.cpp | 11 ++ 12 files changed, 503 insertions(+), 87 deletions(-) create mode 100644 llvm/test/Bitcode/dbg-record-roundtrip.ll diff --git a/llvm/include/llvm/Bitcode/LLVMBitCodes.h b/llvm/include/llvm/Bitcode/LLVMBitCodes.h index c0a52d64a101..39303e648521 100644 --- a/llvm/include/llvm/Bitcode/LLVMBitCodes.h +++ b/llvm/include/llvm/Bitcode/LLVMBitCodes.h @@ -624,6 +624,17 @@ enum FunctionCodes { // operation, align, vol, // ordering, synchscope] FUNC_CODE_BLOCKADDR_USERS = 60, // BLOCKADDR_USERS: [value...] + + FUNC_CODE_DEBUG_RECORD_VALUE = + 61, // [DILocation, DILocalVariable, DIExpression, ValueAsMetadata] + FUNC_CODE_DEBUG_RECORD_DECLARE = + 62, // [DILocation, DILocalVariable, DIExpression, ValueAsMetadata] + FUNC_CODE_DEBUG_RECORD_ASSIGN = + 63, // [DILocation, DILocalVariable, DIExpression, ValueAsMetadata, + // DIAssignID, DIExpression (addr), ValueAsMetadata (addr)] + FUNC_CODE_DEBUG_RECORD_VALUE_SIMPLE = + 64, // [DILocation, DILocalVariable, DIExpression, Value] + FUNC_CODE_DEBUG_RECORD_LABEL = 65, // [DILocation, DILabel] }; enum UseListCodes { diff --git a/llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp b/llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp index 7005011980eb..c085c715179b 100644 --- a/llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp +++ b/llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp @@ -270,6 +270,11 @@ GetCodeName(unsigned CodeID, unsigned BlockID, STRINGIFY_CODE(FUNC_CODE, INST_CMPXCHG) STRINGIFY_CODE(FUNC_CODE, INST_CALLBR) STRINGIFY_CODE(FUNC_CODE, BLOCKADDR_USERS) + STRINGIFY_CODE(FUNC_CODE, DEBUG_RECORD_DECLARE) + STRINGIFY_CODE(FUNC_CODE, DEBUG_RECORD_VALUE) + STRINGIFY_CODE(FUNC_CODE, DEBUG_RECORD_ASSIGN) + STRINGIFY_CODE(FUNC_CODE, DEBUG_RECORD_VALUE_SIMPLE) + STRINGIFY_CODE(FUNC_CODE, DEBUG_RECORD_LABEL) } case bitc::VALUE_SYMTAB_BLOCK_ID: switch (CodeID) { diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp index 9c63116114f3..d284c9823c9e 100644 --- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp +++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp @@ -100,9 +100,6 @@ static cl::opt ExpandConstantExprs( cl::desc( "Expand constant expressions to instructions for testing purposes")); -// Declare external flag for whether we're using the new debug-info format. -extern llvm::cl::opt UseNewDbgInfoFormat; - namespace { enum { @@ -4279,6 +4276,10 @@ Error BitcodeReader::parseGlobalIndirectSymbolRecord( Error BitcodeReader::parseModule(uint64_t ResumeBit, bool ShouldLazyLoadMetadata, ParserCallbacks Callbacks) { + // Force the debug-info mode into the old format for now. + // FIXME: Remove this once all tools support RemoveDIs. + TheModule->IsNewDbgInfoFormat = false; + this->ValueTypeCallback = std::move(Callbacks.ValueType); if (ResumeBit) { if (Error JumpFailed = Stream.JumpToBit(ResumeBit)) @@ -6398,6 +6399,89 @@ Error BitcodeReader::parseFunctionBody(Function *F) { InstructionList.push_back(I); break; } + case bitc::FUNC_CODE_DEBUG_RECORD_LABEL: { + // DPLabels are placed after the Instructions that they are attached to. + Instruction *Inst = getLastInstruction(); + if (!Inst) + return error("Invalid dbg record: missing instruction"); + DILocation *DIL = cast(getFnMetadataByID(Record[0])); + DILabel *Label = cast(getFnMetadataByID(Record[1])); + Inst->getParent()->insertDbgRecordBefore( + new DPLabel(Label, DebugLoc(DIL)), Inst->getIterator()); + continue; // This isn't an instruction. + } + case bitc::FUNC_CODE_DEBUG_RECORD_VALUE_SIMPLE: + case bitc::FUNC_CODE_DEBUG_RECORD_VALUE: + case bitc::FUNC_CODE_DEBUG_RECORD_DECLARE: + case bitc::FUNC_CODE_DEBUG_RECORD_ASSIGN: { + // DPValues are placed after the Instructions that they are attached to. + Instruction *Inst = getLastInstruction(); + if (!Inst) + return error("Invalid dbg record: missing instruction"); + + // First 3 fields are common to all kinds: + // DILocation, DILocalVariable, DIExpression + // dbg_value (FUNC_CODE_DEBUG_RECORD_VALUE) + // ..., LocationMetadata + // dbg_value (FUNC_CODE_DEBUG_RECORD_VALUE_SIMPLE - abbrev'd) + // ..., Value + // dbg_declare (FUNC_CODE_DEBUG_RECORD_DECLARE) + // ..., LocationMetadata + // dbg_assign (FUNC_CODE_DEBUG_RECORD_ASSIGN) + // ..., LocationMetadata, DIAssignID, DIExpression, LocationMetadata + unsigned Slot = 0; + // Common fields (0-2). + DILocation *DIL = cast(getFnMetadataByID(Record[Slot++])); + DILocalVariable *Var = + cast(getFnMetadataByID(Record[Slot++])); + DIExpression *Expr = + cast(getFnMetadataByID(Record[Slot++])); + + // Union field (3: LocationMetadata | Value). + Metadata *RawLocation = nullptr; + if (BitCode == bitc::FUNC_CODE_DEBUG_RECORD_VALUE_SIMPLE) { + Value *V = nullptr; + unsigned TyID = 0; + // We never expect to see a fwd reference value here because + // use-before-defs are encoded with the standard non-abbrev record + // type (they'd require encoding the type too, and they're rare). As a + // result, getValueTypePair only ever increments Slot by one here (once + // for the value, never twice for value and type). + unsigned SlotBefore = Slot; + if (getValueTypePair(Record, Slot, NextValueNo, V, TyID, CurBB)) + return error("Invalid dbg record: invalid value"); + (void)SlotBefore; + assert((SlotBefore == Slot - 1) && "unexpected fwd ref"); + RawLocation = ValueAsMetadata::get(V); + } else { + RawLocation = getFnMetadataByID(Record[Slot++]); + } + + DPValue *DPV = nullptr; + switch (BitCode) { + case bitc::FUNC_CODE_DEBUG_RECORD_VALUE: + case bitc::FUNC_CODE_DEBUG_RECORD_VALUE_SIMPLE: + DPV = new DPValue(RawLocation, Var, Expr, DIL, + DPValue::LocationType::Value); + break; + case bitc::FUNC_CODE_DEBUG_RECORD_DECLARE: + DPV = new DPValue(RawLocation, Var, Expr, DIL, + DPValue::LocationType::Declare); + break; + case bitc::FUNC_CODE_DEBUG_RECORD_ASSIGN: { + DIAssignID *ID = cast(getFnMetadataByID(Record[Slot++])); + DIExpression *AddrExpr = + cast(getFnMetadataByID(Record[Slot++])); + Metadata *Addr = getFnMetadataByID(Record[Slot++]); + DPV = new DPValue(RawLocation, Var, Expr, ID, Addr, AddrExpr, DIL); + break; + } + default: + llvm_unreachable("Unknown DPValue bitcode"); + } + Inst->getParent()->insertDbgRecordBefore(DPV, Inst->getIterator()); + continue; // This isn't an instruction. + } case bitc::FUNC_CODE_INST_CALL: { // CALL: [paramattrs, cc, fmf, fnty, fnid, arg0, arg1...] if (Record.size() < 3) @@ -6677,10 +6761,21 @@ Error BitcodeReader::materialize(GlobalValue *GV) { // Move the bit stream to the saved position of the deferred function body. if (Error JumpFailed = Stream.JumpToBit(DFII->second)) return JumpFailed; + + // Set the debug info mode to "new", forcing a mismatch between + // module and function debug modes. This is okay because we'll convert + // everything back to the old mode after parsing. + // FIXME: Remove this once all tools support RemoveDIs. + F->IsNewDbgInfoFormat = true; + if (Error Err = parseFunctionBody(F)) return Err; F->setIsMaterializable(false); + // Convert new debug info records into intrinsics. + // FIXME: Remove this once all tools support RemoveDIs. + F->convertFromNewDbgValues(); + if (StripDebugInfo) stripDebugInfo(*F); diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp index 597f49332fad..6f0879a4e0ee 100644 --- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp +++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp @@ -99,6 +99,9 @@ namespace llvm { extern FunctionSummary::ForceSummaryHotnessType ForceSummaryEdgesCold; } +extern bool WriteNewDbgInfoFormatToBitcode; +extern llvm::cl::opt UseNewDbgInfoFormat; + namespace { /// These are manifest constants used by the bitcode writer. They do not need to @@ -128,6 +131,7 @@ enum { FUNCTION_INST_RET_VAL_ABBREV, FUNCTION_INST_UNREACHABLE_ABBREV, FUNCTION_INST_GEP_ABBREV, + FUNCTION_DEBUG_RECORD_VALUE_ABBREV, }; /// Abstract class to manage the bitcode writing, subclassed for each bitcode @@ -3512,25 +3516,95 @@ void ModuleBitcodeWriter::writeFunction( NeedsMetadataAttachment |= I.hasMetadataOtherThanDebugLoc(); // If the instruction has a debug location, emit it. - DILocation *DL = I.getDebugLoc(); - if (!DL) - continue; - - if (DL == LastDL) { - // Just repeat the same debug loc as last time. - Stream.EmitRecord(bitc::FUNC_CODE_DEBUG_LOC_AGAIN, Vals); - continue; + if (DILocation *DL = I.getDebugLoc()) { + if (DL == LastDL) { + // Just repeat the same debug loc as last time. + Stream.EmitRecord(bitc::FUNC_CODE_DEBUG_LOC_AGAIN, Vals); + } else { + Vals.push_back(DL->getLine()); + Vals.push_back(DL->getColumn()); + Vals.push_back(VE.getMetadataOrNullID(DL->getScope())); + Vals.push_back(VE.getMetadataOrNullID(DL->getInlinedAt())); + Vals.push_back(DL->isImplicitCode()); + Stream.EmitRecord(bitc::FUNC_CODE_DEBUG_LOC, Vals); + Vals.clear(); + LastDL = DL; + } } - Vals.push_back(DL->getLine()); - Vals.push_back(DL->getColumn()); - Vals.push_back(VE.getMetadataOrNullID(DL->getScope())); - Vals.push_back(VE.getMetadataOrNullID(DL->getInlinedAt())); - Vals.push_back(DL->isImplicitCode()); - Stream.EmitRecord(bitc::FUNC_CODE_DEBUG_LOC, Vals); - Vals.clear(); - - LastDL = DL; + // If the instruction has DbgRecords attached to it, emit them. Note that + // they come after the instruction so that it's easy to attach them again + // when reading the bitcode, even though conceptually the debug locations + // start "before" the instruction. + if (I.hasDbgRecords() && WriteNewDbgInfoFormatToBitcode) { + /// Try to push the value only (unwrapped), otherwise push the + /// metadata wrapped value. Returns true if the value was pushed + /// without the ValueAsMetadata wrapper. + auto PushValueOrMetadata = [&Vals, InstID, + this](Metadata *RawLocation) { + assert(RawLocation && "RawLocation unexpectedly null in DPValue"); + if (ValueAsMetadata *VAM = dyn_cast(RawLocation)) { + SmallVector ValAndType; + // If the value is a fwd-ref the type is also pushed. We don't + // want the type, so fwd-refs are kept wrapped (pushValueAndType + // returns false if the value is pushed without type). + if (!pushValueAndType(VAM->getValue(), InstID, ValAndType)) { + Vals.push_back(ValAndType[0]); + return true; + } + } + // The metadata is a DIArgList, or ValueAsMetadata wrapping a + // fwd-ref. Push the metadata ID. + Vals.push_back(VE.getMetadataID(RawLocation)); + return false; + }; + + // Write out non-instruction debug information attached to this + // instruction. Write it after the instruction so that it's easy to + // re-attach to the instruction reading the records in. + for (DbgRecord &DR : I.DbgMarker->getDbgRecordRange()) { + if (DPLabel *DPL = dyn_cast(&DR)) { + Vals.push_back(VE.getMetadataID(&*DPL->getDebugLoc())); + Vals.push_back(VE.getMetadataID(DPL->getLabel())); + Stream.EmitRecord(bitc::FUNC_CODE_DEBUG_RECORD_LABEL, Vals); + Vals.clear(); + continue; + } + + // First 3 fields are common to all kinds: + // DILocation, DILocalVariable, DIExpression + // dbg_value (FUNC_CODE_DEBUG_RECORD_VALUE) + // ..., LocationMetadata + // dbg_value (FUNC_CODE_DEBUG_RECORD_VALUE_SIMPLE - abbrev'd) + // ..., Value + // dbg_declare (FUNC_CODE_DEBUG_RECORD_DECLARE) + // ..., LocationMetadata + // dbg_assign (FUNC_CODE_DEBUG_RECORD_ASSIGN) + // ..., LocationMetadata, DIAssignID, DIExpression, LocationMetadata + DPValue &DPV = cast(DR); + Vals.push_back(VE.getMetadataID(&*DPV.getDebugLoc())); + Vals.push_back(VE.getMetadataID(DPV.getVariable())); + Vals.push_back(VE.getMetadataID(DPV.getExpression())); + if (DPV.isDbgValue()) { + if (PushValueOrMetadata(DPV.getRawLocation())) + Stream.EmitRecord(bitc::FUNC_CODE_DEBUG_RECORD_VALUE_SIMPLE, Vals, + FUNCTION_DEBUG_RECORD_VALUE_ABBREV); + else + Stream.EmitRecord(bitc::FUNC_CODE_DEBUG_RECORD_VALUE, Vals); + } else if (DPV.isDbgDeclare()) { + Vals.push_back(VE.getMetadataID(DPV.getRawLocation())); + Stream.EmitRecord(bitc::FUNC_CODE_DEBUG_RECORD_DECLARE, Vals); + } else { + assert(DPV.isDbgAssign() && "Unexpected DbgRecord kind"); + Vals.push_back(VE.getMetadataID(DPV.getRawLocation())); + Vals.push_back(VE.getMetadataID(DPV.getAssignID())); + Vals.push_back(VE.getMetadataID(DPV.getAddressExpression())); + Vals.push_back(VE.getMetadataID(DPV.getRawAddress())); + Stream.EmitRecord(bitc::FUNC_CODE_DEBUG_RECORD_ASSIGN, Vals); + } + Vals.clear(); + } + } } if (BlockAddress *BA = BlockAddress::lookup(&BB)) { @@ -3771,7 +3845,17 @@ void ModuleBitcodeWriter::writeBlockInfo() { FUNCTION_INST_GEP_ABBREV) llvm_unreachable("Unexpected abbrev ordering!"); } - + { + auto Abbv = std::make_shared(); + Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_DEBUG_RECORD_VALUE_SIMPLE)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 7)); // dbgloc + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 7)); // var + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 7)); // expr + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // val + if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) != + FUNCTION_DEBUG_RECORD_VALUE_ABBREV) + llvm_unreachable("Unexpected abbrev ordering! 1"); + } Stream.ExitBlock(); } diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriterPass.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriterPass.cpp index 0eb9c246f2a9..de2396f31f66 100644 --- a/llvm/lib/Bitcode/Writer/BitcodeWriterPass.cpp +++ b/llvm/lib/Bitcode/Writer/BitcodeWriterPass.cpp @@ -18,11 +18,12 @@ #include "llvm/Pass.h" using namespace llvm; +extern bool WriteNewDbgInfoFormatToBitcode; + PreservedAnalyses BitcodeWriterPass::run(Module &M, ModuleAnalysisManager &AM) { - // RemoveDIs: there's no bitcode representation of the DbgRecord debug-info, - // convert to dbg.values before writing out. - bool IsNewDbgInfoFormat = M.IsNewDbgInfoFormat; - if (IsNewDbgInfoFormat) + bool ConvertToOldDbgFormatForWrite = + M.IsNewDbgInfoFormat && !WriteNewDbgInfoFormatToBitcode; + if (ConvertToOldDbgFormatForWrite) M.convertFromNewDbgValues(); const ModuleSummaryIndex *Index = @@ -30,7 +31,7 @@ PreservedAnalyses BitcodeWriterPass::run(Module &M, ModuleAnalysisManager &AM) { : nullptr; WriteBitcodeToFile(M, OS, ShouldPreserveUseListOrder, Index, EmitModuleHash); - if (IsNewDbgInfoFormat) + if (ConvertToOldDbgFormatForWrite) M.convertToNewDbgValues(); return PreservedAnalyses::all(); @@ -56,16 +57,15 @@ namespace { StringRef getPassName() const override { return "Bitcode Writer"; } bool runOnModule(Module &M) override { - // RemoveDIs: there's no bitcode representation of the DbgRecord - // debug-info, convert to dbg.values before writing out. - bool IsNewDbgInfoFormat = M.IsNewDbgInfoFormat; - if (IsNewDbgInfoFormat) + bool ConvertToOldDbgFormatForWrite = + M.IsNewDbgInfoFormat && !WriteNewDbgInfoFormatToBitcode; + if (ConvertToOldDbgFormatForWrite) M.convertFromNewDbgValues(); WriteBitcodeToFile(M, OS, ShouldPreserveUseListOrder, /*Index=*/nullptr, /*EmitModuleHash=*/false); - if (IsNewDbgInfoFormat) + if (ConvertToOldDbgFormatForWrite) M.convertToNewDbgValues(); return false; } diff --git a/llvm/lib/Bitcode/Writer/ValueEnumerator.cpp b/llvm/lib/Bitcode/Writer/ValueEnumerator.cpp index fccb2a606f7e..1c439c954321 100644 --- a/llvm/lib/Bitcode/Writer/ValueEnumerator.cpp +++ b/llvm/lib/Bitcode/Writer/ValueEnumerator.cpp @@ -134,20 +134,28 @@ static OrderMap orderModule(const Module &M) { // Metadata used by instructions is decoded before the actual instructions, // so visit any constants used by it beforehand. for (const BasicBlock &BB : F) - for (const Instruction &I : BB) - for (const Value *V : I.operands()) { - if (const auto *MAV = dyn_cast(V)) { - if (const auto *VAM = - dyn_cast(MAV->getMetadata())) { + for (const Instruction &I : BB) { + auto OrderConstantFromMetadata = [&](Metadata *MD) { + if (const auto *VAM = dyn_cast(MD)) { + orderConstantValue(VAM->getValue()); + } else if (const auto *AL = dyn_cast(MD)) { + for (const auto *VAM : AL->getArgs()) orderConstantValue(VAM->getValue()); - } else if (const auto *AL = - dyn_cast(MAV->getMetadata())) { - for (const auto *VAM : AL->getArgs()) - orderConstantValue(VAM->getValue()); - } } + }; + + for (DPValue &DPV : filterDbgVars(I.getDbgRecordRange())) { + OrderConstantFromMetadata(DPV.getRawLocation()); + if (DPV.isDbgAssign()) + OrderConstantFromMetadata(DPV.getRawAddress()); } + for (const Value *V : I.operands()) { + if (const auto *MAV = dyn_cast(V)) + OrderConstantFromMetadata(MAV->getMetadata()); + } + } + for (const Argument &A : F.args()) orderValue(&A, OM); for (const BasicBlock &BB : F) @@ -261,33 +269,39 @@ static UseListOrderStack predictUseListOrder(const Module &M) { // constants in the last Function they're used in. Module-level constants // have already been visited above. for (const Function &F : llvm::reverse(M)) { + auto PredictValueOrderFromMetadata = [&](Metadata *MD) { + if (const auto *VAM = dyn_cast(MD)) { + predictValueUseListOrder(VAM->getValue(), &F, OM, Stack); + } else if (const auto *AL = dyn_cast(MD)) { + for (const auto *VAM : AL->getArgs()) + predictValueUseListOrder(VAM->getValue(), &F, OM, Stack); + } + }; if (F.isDeclaration()) continue; for (const BasicBlock &BB : F) predictValueUseListOrder(&BB, &F, OM, Stack); for (const Argument &A : F.args()) predictValueUseListOrder(&A, &F, OM, Stack); - for (const BasicBlock &BB : F) + for (const BasicBlock &BB : F) { for (const Instruction &I : BB) { + for (DPValue &DPV : filterDbgVars(I.getDbgRecordRange())) { + PredictValueOrderFromMetadata(DPV.getRawLocation()); + if (DPV.isDbgAssign()) + PredictValueOrderFromMetadata(DPV.getRawAddress()); + } for (const Value *Op : I.operands()) { if (isa(*Op) || isa(*Op)) // Visit GlobalValues. predictValueUseListOrder(Op, &F, OM, Stack); - if (const auto *MAV = dyn_cast(Op)) { - if (const auto *VAM = - dyn_cast(MAV->getMetadata())) { - predictValueUseListOrder(VAM->getValue(), &F, OM, Stack); - } else if (const auto *AL = - dyn_cast(MAV->getMetadata())) { - for (const auto *VAM : AL->getArgs()) - predictValueUseListOrder(VAM->getValue(), &F, OM, Stack); - } - } + if (const auto *MAV = dyn_cast(Op)) + PredictValueOrderFromMetadata(MAV->getMetadata()); } if (auto *SVI = dyn_cast(&I)) predictValueUseListOrder(SVI->getShuffleMaskForBitcode(), &F, OM, Stack); predictValueUseListOrder(&I, &F, OM, Stack); } + } } // Visit globals last, since the module-level use-list block will be seen @@ -409,6 +423,41 @@ ValueEnumerator::ValueEnumerator(const Module &M, for (const BasicBlock &BB : F) for (const Instruction &I : BB) { + // Local metadata is enumerated during function-incorporation, but + // any ConstantAsMetadata arguments in a DIArgList should be examined + // now. + auto EnumerateNonLocalValuesFromMetadata = [&](Metadata *MD) { + assert(MD && "Metadata unexpectedly null"); + if (const auto *AL = dyn_cast(MD)) { + for (const auto *VAM : AL->getArgs()) { + if (isa(VAM)) + EnumerateMetadata(&F, VAM); + } + return; + } + + if (!isa(MD)) + EnumerateMetadata(&F, MD); + }; + + for (DbgRecord &DR : I.getDbgRecordRange()) { + if (DPLabel *DPL = dyn_cast(&DR)) { + EnumerateMetadata(&F, DPL->getLabel()); + EnumerateMetadata(&F, &*DPL->getDebugLoc()); + continue; + } + // Enumerate non-local location metadata. + DPValue &DPV = cast(DR); + EnumerateNonLocalValuesFromMetadata(DPV.getRawLocation()); + EnumerateMetadata(&F, DPV.getExpression()); + EnumerateMetadata(&F, DPV.getVariable()); + EnumerateMetadata(&F, &*DPV.getDebugLoc()); + if (DPV.isDbgAssign()) { + EnumerateNonLocalValuesFromMetadata(DPV.getRawAddress()); + EnumerateMetadata(&F, DPV.getAssignID()); + EnumerateMetadata(&F, DPV.getAddressExpression()); + } + } for (const Use &Op : I.operands()) { auto *MD = dyn_cast(&Op); if (!MD) { @@ -416,19 +465,7 @@ ValueEnumerator::ValueEnumerator(const Module &M, continue; } - // Local metadata is enumerated during function-incorporation, but - // any ConstantAsMetadata arguments in a DIArgList should be examined - // now. - if (isa(MD->getMetadata())) - continue; - if (auto *AL = dyn_cast(MD->getMetadata())) { - for (auto *VAM : AL->getArgs()) - if (isa(VAM)) - EnumerateMetadata(&F, VAM); - continue; - } - - EnumerateMetadata(&F, MD->getMetadata()); + EnumerateNonLocalValuesFromMetadata(MD->getMetadata()); } if (auto *SVI = dyn_cast(&I)) EnumerateType(SVI->getShuffleMaskForBitcode()->getType()); @@ -1064,27 +1101,41 @@ void ValueEnumerator::incorporateFunction(const Function &F) { SmallVector FnLocalMDVector; SmallVector ArgListMDVector; + + auto AddFnLocalMetadata = [&](Metadata *MD) { + if (!MD) + return; + if (auto *Local = dyn_cast(MD)) { + // Enumerate metadata after the instructions they might refer to. + FnLocalMDVector.push_back(Local); + } else if (auto *ArgList = dyn_cast(MD)) { + ArgListMDVector.push_back(ArgList); + for (ValueAsMetadata *VMD : ArgList->getArgs()) { + if (auto *Local = dyn_cast(VMD)) { + // Enumerate metadata after the instructions they might refer + // to. + FnLocalMDVector.push_back(Local); + } + } + } + }; + // Add all of the instructions. for (const BasicBlock &BB : F) { for (const Instruction &I : BB) { for (const Use &OI : I.operands()) { - if (auto *MD = dyn_cast(&OI)) { - if (auto *Local = dyn_cast(MD->getMetadata())) { - // Enumerate metadata after the instructions they might refer to. - FnLocalMDVector.push_back(Local); - } else if (auto *ArgList = dyn_cast(MD->getMetadata())) { - ArgListMDVector.push_back(ArgList); - for (ValueAsMetadata *VMD : ArgList->getArgs()) { - if (auto *Local = dyn_cast(VMD)) { - // Enumerate metadata after the instructions they might refer - // to. - FnLocalMDVector.push_back(Local); - } - } - } + if (auto *MD = dyn_cast(&OI)) + AddFnLocalMetadata(MD->getMetadata()); + } + /// RemoveDIs: Add non-instruction function-local metadata uses. + for (DPValue &DPV : filterDbgVars(I.getDbgRecordRange())) { + assert(DPV.getRawLocation() && "DPValue location unexpectedly null"); + AddFnLocalMetadata(DPV.getRawLocation()); + if (DPV.isDbgAssign()) { + assert(DPV.getRawAddress() && "DPValue location unexpectedly null"); + AddFnLocalMetadata(DPV.getRawAddress()); } } - if (!I.getType()->isVoidTy()) EnumerateValue(&I); } diff --git a/llvm/lib/IR/BasicBlock.cpp b/llvm/lib/IR/BasicBlock.cpp index 4dd1bdd6e2f4..5f0780847938 100644 --- a/llvm/lib/IR/BasicBlock.cpp +++ b/llvm/lib/IR/BasicBlock.cpp @@ -36,6 +36,11 @@ cl::opt "through iterators, eliminating intrinsics"), cl::init(true)); +bool WriteNewDbgInfoFormatToBitcode /*set default value in cl::init() below*/; +cl::opt WriteNewDbgInfoFormatToBitcode2( + "write-experimental-debuginfo-iterators-to-bitcode", cl::Hidden, + cl::location(WriteNewDbgInfoFormatToBitcode), cl::init(false)); + DPMarker *BasicBlock::createMarker(Instruction *I) { assert(IsNewDbgInfoFormat && "Tried to create a marker in a non new debug-info block!"); diff --git a/llvm/lib/Linker/IRMover.cpp b/llvm/lib/Linker/IRMover.cpp index 37d21119447b..a7e6db82e5c2 100644 --- a/llvm/lib/Linker/IRMover.cpp +++ b/llvm/lib/Linker/IRMover.cpp @@ -8,6 +8,7 @@ #include "llvm/Linker/IRMover.h" #include "LinkDiagnosticInfo.h" +#include "llvm/ADT/ScopeExit.h" #include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallString.h" @@ -1546,7 +1547,23 @@ Error IRLinker::run() { if (Error Err = SrcM->getMaterializer()->materializeMetadata()) return Err; - DstM.IsNewDbgInfoFormat = SrcM->IsNewDbgInfoFormat; + // Convert source module to match dest for the duration of the link. + bool SrcModuleNewDbgFormat = SrcM->IsNewDbgInfoFormat; + if (DstM.IsNewDbgInfoFormat != SrcM->IsNewDbgInfoFormat) { + if (DstM.IsNewDbgInfoFormat) + SrcM->convertToNewDbgValues(); + else + SrcM->convertFromNewDbgValues(); + } + // Undo debug mode conversion afterwards. + auto Cleanup = make_scope_exit([&]() { + if (SrcModuleNewDbgFormat != SrcM->IsNewDbgInfoFormat) { + if (SrcModuleNewDbgFormat) + SrcM->convertToNewDbgValues(); + else + SrcM->convertFromNewDbgValues(); + } + }); // Inherit the target data from the source module if the destination module // doesn't have one already. @@ -1775,8 +1792,6 @@ IRMover::IRMover(Module &M) : Composite(M) { Error IRMover::move(std::unique_ptr Src, ArrayRef ValuesToLink, LazyCallback AddLazyFor, bool IsPerformingImport) { - if (getModule().IsNewDbgInfoFormat) - Src->convertToNewDbgValues(); IRLinker TheIRLinker(Composite, SharedMDs, IdentifiedStructTypes, std::move(Src), ValuesToLink, std::move(AddLazyFor), IsPerformingImport); diff --git a/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp b/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp index dd6062d303d4..6b9a1f6777fc 100644 --- a/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp +++ b/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp @@ -575,7 +575,7 @@ bool writeThinLTOBitcode(raw_ostream &OS, raw_ostream *ThinLinkOS, } } // anonymous namespace - +extern bool WriteNewDbgInfoFormatToBitcode; PreservedAnalyses llvm::ThinLTOBitcodeWriterPass::run(Module &M, ModuleAnalysisManager &AM) { FunctionAnalysisManager &FAM = @@ -583,8 +583,9 @@ llvm::ThinLTOBitcodeWriterPass::run(Module &M, ModuleAnalysisManager &AM) { // RemoveDIs: there's no bitcode representation of the DPValue debug-info, // convert to dbg.values before writing out. - bool IsNewDbgInfoFormat = M.IsNewDbgInfoFormat; - if (IsNewDbgInfoFormat) + bool ConvertToOldDbgFormatForWrite = + M.IsNewDbgInfoFormat && !WriteNewDbgInfoFormatToBitcode; + if (ConvertToOldDbgFormatForWrite) M.convertFromNewDbgValues(); bool Changed = writeThinLTOBitcode( @@ -594,7 +595,7 @@ llvm::ThinLTOBitcodeWriterPass::run(Module &M, ModuleAnalysisManager &AM) { }, M, &AM.getResult(M)); - if (IsNewDbgInfoFormat) + if (ConvertToOldDbgFormatForWrite) M.convertToNewDbgValues(); return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all(); diff --git a/llvm/test/Bitcode/dbg-record-roundtrip.ll b/llvm/test/Bitcode/dbg-record-roundtrip.ll new file mode 100644 index 000000000000..e7774a9d80d9 --- /dev/null +++ b/llvm/test/Bitcode/dbg-record-roundtrip.ll @@ -0,0 +1,131 @@ +;; Roundtrip tests. +; RUN: llvm-as --write-experimental-debuginfo-iterators-to-bitcode=true %s -o - | llvm-dis | FileCheck %s +;; Check that verify-uselistorder passes regardless of input format. +;; NOTE: This test fails intermittently +; RUN: llvm-as %s --write-experimental-debuginfo-iterators-to-bitcode=true -o - | verify-uselistorder %s +; RUN: verify-uselistorder %s + +;; Confirm we're producing RemoveDI records from various tools. +; RUN: opt %s -o - --write-experimental-debuginfo-iterators-to-bitcode=true | llvm-bcanalyzer - | FileCheck %s --check-prefix=BITCODE +; RUN: llvm-as %s -o - --write-experimental-debuginfo-iterators-to-bitcode=true | llvm-bcanalyzer - | FileCheck %s --check-prefix=BITCODE +; BITCODE-DAG: DEBUG_RECORD_LABEL +; BITCODE-DAG: DEBUG_RECORD_VALUE +; BITCODE-DAG: DEBUG_RECORD_ASSIGN +; BITCODE-DAG: DEBUG_RECORD_DECLARE + +;; Check that llvm-link doesn't explode if we give it different formats to +;; link. +; RUN: llvm-as %s --experimental-debuginfo-iterators=true --write-experimental-debuginfo-iterators-to-bitcode=true -o %t +; RUN: llvm-link %t %s --experimental-debuginfo-iterators=false -o /dev/null +; RUN: llvm-as %s --experimental-debuginfo-iterators=false -o %t +; RUN: llvm-link %t %s --experimental-debuginfo-iterators=true + +;; Checks inline. + +@g = internal dso_local global i32 0, align 4, !dbg !0 + +define internal dso_local noundef i32 @_Z3funv(i32 %p, ptr %storage) !dbg !13 { +entry: +;; Dbg record at top of block, check dbg.value configurations. +; CHECK: entry: +; CHECK-NEXT: dbg.value(metadata i32 %p, metadata ![[e:[0-9]+]], metadata !DIExpression()), !dbg ![[dbg:[0-9]+]] +; CHECK-NEXT: dbg.value(metadata ![[empty:[0-9]+]], metadata ![[e]], metadata !DIExpression()), !dbg ![[dbg]] +; CHECK-NEXT: dbg.value(metadata i32 poison, metadata ![[e]], metadata !DIExpression()), !dbg ![[dbg]] +; CHECK-NEXT: dbg.value(metadata i32 1, metadata ![[f:[0-9]+]], metadata !DIExpression()), !dbg ![[dbg]] + tail call void @llvm.dbg.value(metadata i32 %p, metadata !32, metadata !DIExpression()), !dbg !19 + tail call void @llvm.dbg.value(metadata !29, metadata !32, metadata !DIExpression()), !dbg !19 + tail call void @llvm.dbg.value(metadata i32 poison, metadata !32, metadata !DIExpression()), !dbg !19 + tail call void @llvm.dbg.value(metadata i32 1, metadata !33, metadata !DIExpression()), !dbg !19 +;; Arglist with an argument, constant, local use before def, poison. +; CHECK-NEXT: dbg.value(metadata !DIArgList(i32 %p, i32 0, i32 %0, i32 poison), metadata ![[f]], metadata !DIExpression(DW_OP_LLVM_arg, 0, DW_OP_LLVM_arg, 1, DW_OP_plus, DW_OP_LLVM_arg, 2, DW_OP_LLVM_arg, 3, DW_OP_plus, DW_OP_minus)), !dbg ![[dbg]] + tail call void @llvm.dbg.value(metadata !DIArgList(i32 %p, i32 0, i32 %0, i32 poison), metadata !33, metadata !DIExpression(DW_OP_LLVM_arg, 0, DW_OP_LLVM_arg, 1, DW_OP_plus, DW_OP_LLVM_arg, 2, DW_OP_LLVM_arg, 3, DW_OP_plus, DW_OP_minus)), !dbg !19 +;; Check dbg.assign use before def (value, addr and ID). Check expression order too. +; CHECK: dbg.assign(metadata i32 %0, metadata ![[i:[0-9]+]], metadata !DIExpression(DW_OP_plus_uconst, 0), +; CHECK-SAME: metadata ![[ID:[0-9]+]], metadata ptr %a, metadata !DIExpression(DW_OP_plus_uconst, 1)), !dbg ![[dbg]] + tail call void @llvm.dbg.assign(metadata i32 %0, metadata !36, metadata !DIExpression(DW_OP_plus_uconst, 0), metadata !37, metadata ptr %a, metadata !DIExpression(DW_OP_plus_uconst, 1)), !dbg !19 + %a = alloca i32, align 4, !DIAssignID !37 +; CHECK: %a = alloca i32, align 4, !DIAssignID ![[ID]] +;; Check dbg.declare configurations. +; CHECK-NEXT: dbg.declare(metadata ptr %a, metadata ![[a:[0-9]+]], metadata !DIExpression()), !dbg ![[dbg]] +; CHECK-NEXT: dbg.declare(metadata ![[empty:[0-9]+]], metadata ![[b:[0-9]+]], metadata !DIExpression()), !dbg ![[dbg]] +; CHECK-NEXT: dbg.declare(metadata ptr poison, metadata ![[c:[0-9]+]], metadata !DIExpression()), !dbg ![[dbg]] +; CHECK-NEXT: dbg.declare(metadata ptr null, metadata ![[d:[0-9]+]], metadata !DIExpression()), !dbg ![[dbg]] +; CHECK-NEXT: dbg.declare(metadata ptr @g, metadata ![[h:[0-9]+]], metadata !DIExpression()), !dbg ![[dbg]] + tail call void @llvm.dbg.declare(metadata ptr %a, metadata !17, metadata !DIExpression()), !dbg !19 + tail call void @llvm.dbg.declare(metadata !29, metadata !28, metadata !DIExpression()), !dbg !19 + tail call void @llvm.dbg.declare(metadata ptr poison, metadata !30, metadata !DIExpression()), !dbg !19 + tail call void @llvm.dbg.declare(metadata ptr null, metadata !31, metadata !DIExpression()), !dbg !19 + tail call void @llvm.dbg.declare(metadata ptr @g, metadata !35, metadata !DIExpression()), !dbg !19 +;; Argument value dbg.declare. +; CHECK: dbg.declare(metadata ptr %storage, metadata ![[g:[0-9]+]], metadata !DIExpression()), !dbg ![[dbg]] + tail call void @llvm.dbg.declare(metadata ptr %storage, metadata !34, metadata !DIExpression()), !dbg !19 +;; Use before def dbg.value. +; CHECK: dbg.value(metadata i32 %0, metadata ![[e]], metadata !DIExpression()), !dbg ![[dbg]] + tail call void @llvm.dbg.value(metadata i32 %0, metadata !32, metadata !DIExpression()), !dbg !19 + %0 = load i32, ptr @g, align 4, !dbg !20 +;; Non-argument local value dbg.value. +; CHECK: dbg.value(metadata i32 %0, metadata ![[e]], metadata !DIExpression()), !dbg ![[dbg]] + tail call void @llvm.dbg.value(metadata i32 %0, metadata !32, metadata !DIExpression()), !dbg !19 + store i32 %0, ptr %a, align 4, !dbg !19 + %1 = load i32, ptr %a, align 4, !dbg !25 +; CHECK: dbg.label(metadata ![[label:[0-9]+]]), !dbg ![[dbg]] + tail call void @llvm.dbg.label(metadata !38), !dbg !19 + ret i32 %1, !dbg !27 +} + +; CHECK-DAG: ![[a]] = !DILocalVariable(name: "a", +; CHECK-DAG: ![[b]] = !DILocalVariable(name: "b", +; CHECK-DAG: ![[c]] = !DILocalVariable(name: "c", +; CHECK-DAG: ![[d]] = !DILocalVariable(name: "d", +; CHECK-DAG: ![[e]] = !DILocalVariable(name: "e", +; CHECK-DAG: ![[f]] = !DILocalVariable(name: "f", +; CHECK-DAG: ![[g]] = !DILocalVariable(name: "g", +; CHECK-DAG: ![[h]] = !DILocalVariable(name: "h", +; CHECK-DAG: ![[i]] = !DILocalVariable(name: "i", +; CHECK-DAG: ![[empty]] = !{} +; CHECK-DAG: ![[label]] = !DILabel + +declare void @llvm.dbg.declare(metadata, metadata, metadata) +declare void @llvm.dbg.value(metadata, metadata, metadata) +declare void @llvm.dbg.assign(metadata, metadata, metadata, metadata, metadata, metadata) +declare void @llvm.dbg.label(metadata) + +!llvm.dbg.cu = !{!2} +!llvm.module.flags = !{!6, !7, !8, !9, !10, !11} +!llvm.ident = !{!12} + +!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression()) +!1 = distinct !DIGlobalVariable(name: "g", scope: !2, file: !3, line: 1, type: !5, isLocal: false, isDefinition: true) +!2 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !3, producer: "clang version 19.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, globals: !4, splitDebugInlining: false, nameTableKind: None) +!3 = !DIFile(filename: "test.cpp", directory: "/") +!4 = !{!0} +!5 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) +!6 = !{i32 7, !"Dwarf Version", i32 5} +!7 = !{i32 2, !"Debug Info Version", i32 3} +!8 = !{i32 1, !"wchar_size", i32 4} +!9 = !{i32 8, !"PIC Level", i32 2} +!10 = !{i32 7, !"PIE Level", i32 2} +!11 = !{i32 7, !"uwtable", i32 2} +!12 = !{!"clang version 19.0.0"} +!13 = distinct !DISubprogram(name: "fun", linkageName: "_Z3funv", scope: !3, file: !3, line: 2, type: !14, scopeLine: 2, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2, retainedNodes: !16) +!14 = !DISubroutineType(types: !15) +!15 = !{!5} +!16 = !{!17} +!17 = !DILocalVariable(name: "a", scope: !13, file: !3, line: 3, type: !5) +!18 = !DILocation(line: 3, column: 3, scope: !13) +!19 = !DILocation(line: 3, column: 7, scope: !13) +!20 = !DILocation(line: 3, column: 11, scope: !13) +!25 = !DILocation(line: 4, column: 12, scope: !13) +!26 = !DILocation(line: 5, column: 1, scope: !13) +!27 = !DILocation(line: 4, column: 5, scope: !13) +!28 = !DILocalVariable(name: "b", scope: !13, file: !3, line: 3, type: !5) +!29 = !{} +!30 = !DILocalVariable(name: "c", scope: !13, file: !3, line: 3, type: !5) +!31 = !DILocalVariable(name: "d", scope: !13, file: !3, line: 3, type: !5) +!32 = !DILocalVariable(name: "e", scope: !13, file: !3, line: 3, type: !5) +!33 = !DILocalVariable(name: "f", scope: !13, file: !3, line: 3, type: !5) +!34 = !DILocalVariable(name: "g", scope: !13, file: !3, line: 3, type: !5) +!35 = !DILocalVariable(name: "h", scope: !13, file: !3, line: 3, type: !5) +!36 = !DILocalVariable(name: "i", scope: !13, file: !3, line: 3, type: !5) +!37 = distinct !DIAssignID() +!38 = !DILabel(scope: !13, name: "label", file: !3, line: 1) diff --git a/llvm/tools/llvm-as/llvm-as.cpp b/llvm/tools/llvm-as/llvm-as.cpp index 1c869e173931..fd852563838f 100644 --- a/llvm/tools/llvm-as/llvm-as.cpp +++ b/llvm/tools/llvm-as/llvm-as.cpp @@ -67,6 +67,7 @@ static cl::opt ClDataLayout("data-layout", cl::desc("data layout string to use"), cl::value_desc("layout-string"), cl::init(""), cl::cat(AsCat)); +extern bool WriteNewDbgInfoFormatToBitcode; static void WriteOutputFile(const Module *M, const ModuleSummaryIndex *Index) { // Infer the output filename if needed. @@ -139,6 +140,12 @@ int main(int argc, char **argv) { Err.print(argv[0], errs()); return 1; } + + // Convert to new debug format if requested. + assert(!M->IsNewDbgInfoFormat && "Unexpectedly in new debug mode"); + if (UseNewDbgInfoFormat && WriteNewDbgInfoFormatToBitcode) + M->convertToNewDbgValues(); + std::unique_ptr Index = std::move(ModuleAndIndex.Index); if (!DisableVerify) { diff --git a/llvm/tools/verify-uselistorder/verify-uselistorder.cpp b/llvm/tools/verify-uselistorder/verify-uselistorder.cpp index 9afe6817fefb..d929ae09958a 100644 --- a/llvm/tools/verify-uselistorder/verify-uselistorder.cpp +++ b/llvm/tools/verify-uselistorder/verify-uselistorder.cpp @@ -166,6 +166,11 @@ std::unique_ptr TempFile::readBitcode(LLVMContext &Context) const { "verify-uselistorder: error: "); return nullptr; } + + // verify-uselistoder currently only supports old-style debug info mode. + // FIXME: Update mapping code for RemoveDIs. + assert(!ModuleOr.get()->IsNewDbgInfoFormat && + "Unexpectedly in new debug info mode"); return std::move(ModuleOr.get()); } @@ -175,6 +180,9 @@ std::unique_ptr TempFile::readAssembly(LLVMContext &Context) const { std::unique_ptr M = parseAssemblyFile(Filename, Err, Context); if (!M.get()) Err.print("verify-uselistorder", errs()); + // verify-uselistoder currently only supports old-style debug info mode. + // FIXME: Update mapping code for RemoveDIs. + assert(!M->IsNewDbgInfoFormat && "Unexpectedly in new debug info mode"); return M; } @@ -541,6 +549,9 @@ int main(int argc, char **argv) { // Load the input module... std::unique_ptr M = parseIRFile(InputFilename, Err, Context); + // verify-uselistoder currently only supports old-style debug info mode. + // FIXME: Update mapping code for RemoveDIs. + assert(!M->IsNewDbgInfoFormat && "Unexpectedly in new debug info mode"); if (!M.get()) { Err.print(argv[0], errs()); -- GitLab From 0bcec96f3fe919fc5fd1189d1901fa3a0e501e2c Mon Sep 17 00:00:00 2001 From: Jay Foad Date: Fri, 15 Mar 2024 12:39:10 +0000 Subject: [PATCH 005/514] [AMDGPU] Reduce duplication in FLAT atomic definitions (#85383) This simplifies the case where the tablegen name of the defm for the Real is the same as the name of the Pseudo. --- llvm/lib/Target/AMDGPU/FLATInstructions.td | 143 +++++++++++---------- 1 file changed, 73 insertions(+), 70 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/FLATInstructions.td b/llvm/lib/Target/AMDGPU/FLATInstructions.td index f42d4ae416bd..3ed743542426 100644 --- a/llvm/lib/Target/AMDGPU/FLATInstructions.td +++ b/llvm/lib/Target/AMDGPU/FLATInstructions.td @@ -1787,45 +1787,46 @@ def FLAT_STORE_DWORDX2_ci : FLAT_Real_ci <0x1d, FLAT_STORE_DWORDX2>; def FLAT_STORE_DWORDX4_ci : FLAT_Real_ci <0x1e, FLAT_STORE_DWORDX4>; def FLAT_STORE_DWORDX3_ci : FLAT_Real_ci <0x1f, FLAT_STORE_DWORDX3>; -multiclass FLAT_Real_Atomics_ci op, FLAT_Pseudo ps> { +multiclass FLAT_Real_Atomics_ci op> { + defvar ps = !cast(NAME); def _ci : FLAT_Real_ci(ps.PseudoInstr)>; def _RTN_ci : FLAT_Real_ci(ps.PseudoInstr # "_RTN")>; } -defm FLAT_ATOMIC_SWAP : FLAT_Real_Atomics_ci <0x30, FLAT_ATOMIC_SWAP>; -defm FLAT_ATOMIC_CMPSWAP : FLAT_Real_Atomics_ci <0x31, FLAT_ATOMIC_CMPSWAP>; -defm FLAT_ATOMIC_ADD : FLAT_Real_Atomics_ci <0x32, FLAT_ATOMIC_ADD>; -defm FLAT_ATOMIC_SUB : FLAT_Real_Atomics_ci <0x33, FLAT_ATOMIC_SUB>; -defm FLAT_ATOMIC_SMIN : FLAT_Real_Atomics_ci <0x35, FLAT_ATOMIC_SMIN>; -defm FLAT_ATOMIC_UMIN : FLAT_Real_Atomics_ci <0x36, FLAT_ATOMIC_UMIN>; -defm FLAT_ATOMIC_SMAX : FLAT_Real_Atomics_ci <0x37, FLAT_ATOMIC_SMAX>; -defm FLAT_ATOMIC_UMAX : FLAT_Real_Atomics_ci <0x38, FLAT_ATOMIC_UMAX>; -defm FLAT_ATOMIC_AND : FLAT_Real_Atomics_ci <0x39, FLAT_ATOMIC_AND>; -defm FLAT_ATOMIC_OR : FLAT_Real_Atomics_ci <0x3a, FLAT_ATOMIC_OR>; -defm FLAT_ATOMIC_XOR : FLAT_Real_Atomics_ci <0x3b, FLAT_ATOMIC_XOR>; -defm FLAT_ATOMIC_INC : FLAT_Real_Atomics_ci <0x3c, FLAT_ATOMIC_INC>; -defm FLAT_ATOMIC_DEC : FLAT_Real_Atomics_ci <0x3d, FLAT_ATOMIC_DEC>; -defm FLAT_ATOMIC_SWAP_X2 : FLAT_Real_Atomics_ci <0x50, FLAT_ATOMIC_SWAP_X2>; -defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_Real_Atomics_ci <0x51, FLAT_ATOMIC_CMPSWAP_X2>; -defm FLAT_ATOMIC_ADD_X2 : FLAT_Real_Atomics_ci <0x52, FLAT_ATOMIC_ADD_X2>; -defm FLAT_ATOMIC_SUB_X2 : FLAT_Real_Atomics_ci <0x53, FLAT_ATOMIC_SUB_X2>; -defm FLAT_ATOMIC_SMIN_X2 : FLAT_Real_Atomics_ci <0x55, FLAT_ATOMIC_SMIN_X2>; -defm FLAT_ATOMIC_UMIN_X2 : FLAT_Real_Atomics_ci <0x56, FLAT_ATOMIC_UMIN_X2>; -defm FLAT_ATOMIC_SMAX_X2 : FLAT_Real_Atomics_ci <0x57, FLAT_ATOMIC_SMAX_X2>; -defm FLAT_ATOMIC_UMAX_X2 : FLAT_Real_Atomics_ci <0x58, FLAT_ATOMIC_UMAX_X2>; -defm FLAT_ATOMIC_AND_X2 : FLAT_Real_Atomics_ci <0x59, FLAT_ATOMIC_AND_X2>; -defm FLAT_ATOMIC_OR_X2 : FLAT_Real_Atomics_ci <0x5a, FLAT_ATOMIC_OR_X2>; -defm FLAT_ATOMIC_XOR_X2 : FLAT_Real_Atomics_ci <0x5b, FLAT_ATOMIC_XOR_X2>; -defm FLAT_ATOMIC_INC_X2 : FLAT_Real_Atomics_ci <0x5c, FLAT_ATOMIC_INC_X2>; -defm FLAT_ATOMIC_DEC_X2 : FLAT_Real_Atomics_ci <0x5d, FLAT_ATOMIC_DEC_X2>; +defm FLAT_ATOMIC_SWAP : FLAT_Real_Atomics_ci <0x30>; +defm FLAT_ATOMIC_CMPSWAP : FLAT_Real_Atomics_ci <0x31>; +defm FLAT_ATOMIC_ADD : FLAT_Real_Atomics_ci <0x32>; +defm FLAT_ATOMIC_SUB : FLAT_Real_Atomics_ci <0x33>; +defm FLAT_ATOMIC_SMIN : FLAT_Real_Atomics_ci <0x35>; +defm FLAT_ATOMIC_UMIN : FLAT_Real_Atomics_ci <0x36>; +defm FLAT_ATOMIC_SMAX : FLAT_Real_Atomics_ci <0x37>; +defm FLAT_ATOMIC_UMAX : FLAT_Real_Atomics_ci <0x38>; +defm FLAT_ATOMIC_AND : FLAT_Real_Atomics_ci <0x39>; +defm FLAT_ATOMIC_OR : FLAT_Real_Atomics_ci <0x3a>; +defm FLAT_ATOMIC_XOR : FLAT_Real_Atomics_ci <0x3b>; +defm FLAT_ATOMIC_INC : FLAT_Real_Atomics_ci <0x3c>; +defm FLAT_ATOMIC_DEC : FLAT_Real_Atomics_ci <0x3d>; +defm FLAT_ATOMIC_SWAP_X2 : FLAT_Real_Atomics_ci <0x50>; +defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_Real_Atomics_ci <0x51>; +defm FLAT_ATOMIC_ADD_X2 : FLAT_Real_Atomics_ci <0x52>; +defm FLAT_ATOMIC_SUB_X2 : FLAT_Real_Atomics_ci <0x53>; +defm FLAT_ATOMIC_SMIN_X2 : FLAT_Real_Atomics_ci <0x55>; +defm FLAT_ATOMIC_UMIN_X2 : FLAT_Real_Atomics_ci <0x56>; +defm FLAT_ATOMIC_SMAX_X2 : FLAT_Real_Atomics_ci <0x57>; +defm FLAT_ATOMIC_UMAX_X2 : FLAT_Real_Atomics_ci <0x58>; +defm FLAT_ATOMIC_AND_X2 : FLAT_Real_Atomics_ci <0x59>; +defm FLAT_ATOMIC_OR_X2 : FLAT_Real_Atomics_ci <0x5a>; +defm FLAT_ATOMIC_XOR_X2 : FLAT_Real_Atomics_ci <0x5b>; +defm FLAT_ATOMIC_INC_X2 : FLAT_Real_Atomics_ci <0x5c>; +defm FLAT_ATOMIC_DEC_X2 : FLAT_Real_Atomics_ci <0x5d>; // CI Only flat instructions -defm FLAT_ATOMIC_FCMPSWAP : FLAT_Real_Atomics_ci <0x3e, FLAT_ATOMIC_FCMPSWAP>; -defm FLAT_ATOMIC_FMIN : FLAT_Real_Atomics_ci <0x3f, FLAT_ATOMIC_FMIN>; -defm FLAT_ATOMIC_FMAX : FLAT_Real_Atomics_ci <0x40, FLAT_ATOMIC_FMAX>; -defm FLAT_ATOMIC_FCMPSWAP_X2 : FLAT_Real_Atomics_ci <0x5e, FLAT_ATOMIC_FCMPSWAP_X2>; -defm FLAT_ATOMIC_FMIN_X2 : FLAT_Real_Atomics_ci <0x5f, FLAT_ATOMIC_FMIN_X2>; -defm FLAT_ATOMIC_FMAX_X2 : FLAT_Real_Atomics_ci <0x60, FLAT_ATOMIC_FMAX_X2>; +defm FLAT_ATOMIC_FCMPSWAP : FLAT_Real_Atomics_ci <0x3e>; +defm FLAT_ATOMIC_FMIN : FLAT_Real_Atomics_ci <0x3f>; +defm FLAT_ATOMIC_FMAX : FLAT_Real_Atomics_ci <0x40>; +defm FLAT_ATOMIC_FCMPSWAP_X2 : FLAT_Real_Atomics_ci <0x5e>; +defm FLAT_ATOMIC_FMIN_X2 : FLAT_Real_Atomics_ci <0x5f>; +defm FLAT_ATOMIC_FMAX_X2 : FLAT_Real_Atomics_ci <0x60>; //===----------------------------------------------------------------------===// @@ -1925,8 +1926,9 @@ def FLAT_LOAD_SBYTE_D16_HI_vi : FLAT_Real_vi <0x23, FLAT_LOAD_SBYTE_D16_HI>; def FLAT_LOAD_SHORT_D16_vi : FLAT_Real_vi <0x24, FLAT_LOAD_SHORT_D16>; def FLAT_LOAD_SHORT_D16_HI_vi : FLAT_Real_vi <0x25, FLAT_LOAD_SHORT_D16_HI>; -multiclass FLAT_Real_Atomics_vi op, FLAT_Pseudo ps, +multiclass FLAT_Real_Atomics_vi op, bit has_sccb = !cast(NAME).has_sccb> { + defvar ps = !cast(NAME); def _vi : FLAT_Real_vi(ps.PseudoInstr), has_sccb>; def _RTN_vi : FLAT_Real_vi(ps.PseudoInstr # "_RTN"), has_sccb>; } @@ -1939,32 +1941,32 @@ multiclass FLAT_Global_Real_Atomics_vi op, } -defm FLAT_ATOMIC_SWAP : FLAT_Real_Atomics_vi <0x40, FLAT_ATOMIC_SWAP>; -defm FLAT_ATOMIC_CMPSWAP : FLAT_Real_Atomics_vi <0x41, FLAT_ATOMIC_CMPSWAP>; -defm FLAT_ATOMIC_ADD : FLAT_Real_Atomics_vi <0x42, FLAT_ATOMIC_ADD>; -defm FLAT_ATOMIC_SUB : FLAT_Real_Atomics_vi <0x43, FLAT_ATOMIC_SUB>; -defm FLAT_ATOMIC_SMIN : FLAT_Real_Atomics_vi <0x44, FLAT_ATOMIC_SMIN>; -defm FLAT_ATOMIC_UMIN : FLAT_Real_Atomics_vi <0x45, FLAT_ATOMIC_UMIN>; -defm FLAT_ATOMIC_SMAX : FLAT_Real_Atomics_vi <0x46, FLAT_ATOMIC_SMAX>; -defm FLAT_ATOMIC_UMAX : FLAT_Real_Atomics_vi <0x47, FLAT_ATOMIC_UMAX>; -defm FLAT_ATOMIC_AND : FLAT_Real_Atomics_vi <0x48, FLAT_ATOMIC_AND>; -defm FLAT_ATOMIC_OR : FLAT_Real_Atomics_vi <0x49, FLAT_ATOMIC_OR>; -defm FLAT_ATOMIC_XOR : FLAT_Real_Atomics_vi <0x4a, FLAT_ATOMIC_XOR>; -defm FLAT_ATOMIC_INC : FLAT_Real_Atomics_vi <0x4b, FLAT_ATOMIC_INC>; -defm FLAT_ATOMIC_DEC : FLAT_Real_Atomics_vi <0x4c, FLAT_ATOMIC_DEC>; -defm FLAT_ATOMIC_SWAP_X2 : FLAT_Real_Atomics_vi <0x60, FLAT_ATOMIC_SWAP_X2>; -defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_Real_Atomics_vi <0x61, FLAT_ATOMIC_CMPSWAP_X2>; -defm FLAT_ATOMIC_ADD_X2 : FLAT_Real_Atomics_vi <0x62, FLAT_ATOMIC_ADD_X2>; -defm FLAT_ATOMIC_SUB_X2 : FLAT_Real_Atomics_vi <0x63, FLAT_ATOMIC_SUB_X2>; -defm FLAT_ATOMIC_SMIN_X2 : FLAT_Real_Atomics_vi <0x64, FLAT_ATOMIC_SMIN_X2>; -defm FLAT_ATOMIC_UMIN_X2 : FLAT_Real_Atomics_vi <0x65, FLAT_ATOMIC_UMIN_X2>; -defm FLAT_ATOMIC_SMAX_X2 : FLAT_Real_Atomics_vi <0x66, FLAT_ATOMIC_SMAX_X2>; -defm FLAT_ATOMIC_UMAX_X2 : FLAT_Real_Atomics_vi <0x67, FLAT_ATOMIC_UMAX_X2>; -defm FLAT_ATOMIC_AND_X2 : FLAT_Real_Atomics_vi <0x68, FLAT_ATOMIC_AND_X2>; -defm FLAT_ATOMIC_OR_X2 : FLAT_Real_Atomics_vi <0x69, FLAT_ATOMIC_OR_X2>; -defm FLAT_ATOMIC_XOR_X2 : FLAT_Real_Atomics_vi <0x6a, FLAT_ATOMIC_XOR_X2>; -defm FLAT_ATOMIC_INC_X2 : FLAT_Real_Atomics_vi <0x6b, FLAT_ATOMIC_INC_X2>; -defm FLAT_ATOMIC_DEC_X2 : FLAT_Real_Atomics_vi <0x6c, FLAT_ATOMIC_DEC_X2>; +defm FLAT_ATOMIC_SWAP : FLAT_Real_Atomics_vi <0x40>; +defm FLAT_ATOMIC_CMPSWAP : FLAT_Real_Atomics_vi <0x41>; +defm FLAT_ATOMIC_ADD : FLAT_Real_Atomics_vi <0x42>; +defm FLAT_ATOMIC_SUB : FLAT_Real_Atomics_vi <0x43>; +defm FLAT_ATOMIC_SMIN : FLAT_Real_Atomics_vi <0x44>; +defm FLAT_ATOMIC_UMIN : FLAT_Real_Atomics_vi <0x45>; +defm FLAT_ATOMIC_SMAX : FLAT_Real_Atomics_vi <0x46>; +defm FLAT_ATOMIC_UMAX : FLAT_Real_Atomics_vi <0x47>; +defm FLAT_ATOMIC_AND : FLAT_Real_Atomics_vi <0x48>; +defm FLAT_ATOMIC_OR : FLAT_Real_Atomics_vi <0x49>; +defm FLAT_ATOMIC_XOR : FLAT_Real_Atomics_vi <0x4a>; +defm FLAT_ATOMIC_INC : FLAT_Real_Atomics_vi <0x4b>; +defm FLAT_ATOMIC_DEC : FLAT_Real_Atomics_vi <0x4c>; +defm FLAT_ATOMIC_SWAP_X2 : FLAT_Real_Atomics_vi <0x60>; +defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_Real_Atomics_vi <0x61>; +defm FLAT_ATOMIC_ADD_X2 : FLAT_Real_Atomics_vi <0x62>; +defm FLAT_ATOMIC_SUB_X2 : FLAT_Real_Atomics_vi <0x63>; +defm FLAT_ATOMIC_SMIN_X2 : FLAT_Real_Atomics_vi <0x64>; +defm FLAT_ATOMIC_UMIN_X2 : FLAT_Real_Atomics_vi <0x65>; +defm FLAT_ATOMIC_SMAX_X2 : FLAT_Real_Atomics_vi <0x66>; +defm FLAT_ATOMIC_UMAX_X2 : FLAT_Real_Atomics_vi <0x67>; +defm FLAT_ATOMIC_AND_X2 : FLAT_Real_Atomics_vi <0x68>; +defm FLAT_ATOMIC_OR_X2 : FLAT_Real_Atomics_vi <0x69>; +defm FLAT_ATOMIC_XOR_X2 : FLAT_Real_Atomics_vi <0x6a>; +defm FLAT_ATOMIC_INC_X2 : FLAT_Real_Atomics_vi <0x6b>; +defm FLAT_ATOMIC_DEC_X2 : FLAT_Real_Atomics_vi <0x6c>; defm GLOBAL_LOAD_UBYTE : FLAT_Real_AllAddr_vi <0x10>; defm GLOBAL_LOAD_SBYTE : FLAT_Real_AllAddr_vi <0x11>; @@ -2060,9 +2062,9 @@ let SubtargetPredicate = isGFX8GFX9NotGFX940 in { } let SubtargetPredicate = isGFX90AOnly in { - defm FLAT_ATOMIC_ADD_F64 : FLAT_Real_Atomics_vi<0x4f, FLAT_ATOMIC_ADD_F64, 0>; - defm FLAT_ATOMIC_MIN_F64 : FLAT_Real_Atomics_vi<0x50, FLAT_ATOMIC_MIN_F64, 0>; - defm FLAT_ATOMIC_MAX_F64 : FLAT_Real_Atomics_vi<0x51, FLAT_ATOMIC_MAX_F64, 0>; + defm FLAT_ATOMIC_ADD_F64 : FLAT_Real_Atomics_vi<0x4f, 0>; + defm FLAT_ATOMIC_MIN_F64 : FLAT_Real_Atomics_vi<0x50, 0>; + defm FLAT_ATOMIC_MAX_F64 : FLAT_Real_Atomics_vi<0x51, 0>; defm GLOBAL_ATOMIC_ADD_F64 : FLAT_Global_Real_Atomics_vi<0x4f, 0>; defm GLOBAL_ATOMIC_MIN_F64 : FLAT_Global_Real_Atomics_vi<0x50, 0>; defm GLOBAL_ATOMIC_MAX_F64 : FLAT_Global_Real_Atomics_vi<0x51, 0>; @@ -2073,7 +2075,8 @@ multiclass FLAT_Real_AllAddr_gfx940 op> { def _SADDR_gfx940 : FLAT_Real_gfx940(NAME#"_SADDR")>; } -multiclass FLAT_Real_Atomics_gfx940 op, FLAT_Pseudo ps> { +multiclass FLAT_Real_Atomics_gfx940 op> { + defvar ps = !cast(NAME); def _gfx940 : FLAT_Real_gfx940(ps.PseudoInstr)>; def _RTN_gfx940 : FLAT_Real_gfx940(ps.PseudoInstr # "_RTN")>; } @@ -2089,15 +2092,15 @@ let SubtargetPredicate = isGFX940Plus in { defm GLOBAL_ATOMIC_ADD_F32 : FLAT_Global_Real_Atomics_gfx940 <0x04d>; defm GLOBAL_ATOMIC_PK_ADD_F16 : FLAT_Global_Real_Atomics_gfx940 <0x04e>; - defm FLAT_ATOMIC_ADD_F64 : FLAT_Real_Atomics_gfx940<0x4f, FLAT_ATOMIC_ADD_F64>; - defm FLAT_ATOMIC_MIN_F64 : FLAT_Real_Atomics_gfx940<0x50, FLAT_ATOMIC_MIN_F64>; - defm FLAT_ATOMIC_MAX_F64 : FLAT_Real_Atomics_gfx940<0x51, FLAT_ATOMIC_MAX_F64>; + defm FLAT_ATOMIC_ADD_F64 : FLAT_Real_Atomics_gfx940<0x4f>; + defm FLAT_ATOMIC_MIN_F64 : FLAT_Real_Atomics_gfx940<0x50>; + defm FLAT_ATOMIC_MAX_F64 : FLAT_Real_Atomics_gfx940<0x51>; defm GLOBAL_ATOMIC_ADD_F64 : FLAT_Global_Real_Atomics_gfx940<0x4f>; defm GLOBAL_ATOMIC_MIN_F64 : FLAT_Global_Real_Atomics_gfx940<0x50>; defm GLOBAL_ATOMIC_MAX_F64 : FLAT_Global_Real_Atomics_gfx940<0x51>; - defm FLAT_ATOMIC_ADD_F32 : FLAT_Real_Atomics_vi<0x4d, FLAT_ATOMIC_ADD_F32>; - defm FLAT_ATOMIC_PK_ADD_F16 : FLAT_Real_Atomics_vi<0x4e, FLAT_ATOMIC_PK_ADD_F16>; - defm FLAT_ATOMIC_PK_ADD_BF16 : FLAT_Real_Atomics_vi<0x52, FLAT_ATOMIC_PK_ADD_BF16>; + defm FLAT_ATOMIC_ADD_F32 : FLAT_Real_Atomics_vi<0x4d>; + defm FLAT_ATOMIC_PK_ADD_F16 : FLAT_Real_Atomics_vi<0x4e>; + defm FLAT_ATOMIC_PK_ADD_BF16 : FLAT_Real_Atomics_vi<0x52>; defm GLOBAL_ATOMIC_PK_ADD_BF16 : FLAT_Global_Real_Atomics_vi<0x52>; } // End SubtargetPredicate = isGFX940Plus -- GitLab From 9214e51925d202fd8463535d75f144b81173dd73 Mon Sep 17 00:00:00 2001 From: Jay Foad Date: Fri, 15 Mar 2024 13:07:43 +0000 Subject: [PATCH 006/514] [AMDGPU] Simplify GFX10+ FLAT saddr field definition On GFX10+ has_saddr is effectively always true so there is no need to test it. --- llvm/lib/Target/AMDGPU/FLATInstructions.td | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/FLATInstructions.td b/llvm/lib/Target/AMDGPU/FLATInstructions.td index 3ed743542426..594754492564 100644 --- a/llvm/lib/Target/AMDGPU/FLATInstructions.td +++ b/llvm/lib/Target/AMDGPU/FLATInstructions.td @@ -174,7 +174,7 @@ class VFLAT_Real op, FLAT_Pseudo ps, string opName = ps.Mnemonic> : bits<8> vaddr; bits<24> offset; - let Inst{6-0} = !if(ps.has_saddr, !if(ps.enabled_saddr, saddr, 0x7f), 0); + let Inst{6-0} = !if(ps.enabled_saddr, saddr, 0x7f); let Inst{21-14} = op; let Inst{31-26} = 0x3b; let Inst{39-32} = !if(ps.has_vdst, vdst, ?); @@ -2115,7 +2115,7 @@ class FLAT_Real_gfx10 op, FLAT_Pseudo ps> : let Inst{11-0} = offset{11-0}; let Inst{12} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlcValue); - let Inst{54-48} = !if(ps.has_saddr, !if(ps.enabled_saddr, saddr, 0x7d), 0x7d); + let Inst{54-48} = !if(ps.enabled_saddr, saddr, 0x7d); let Inst{55} = 0; } -- GitLab From e419084da7a00b269368aeb95698e0d36b24e8ec Mon Sep 17 00:00:00 2001 From: Orlando Cazalet-Hyams Date: Fri, 15 Mar 2024 12:41:32 +0000 Subject: [PATCH 007/514] [RemoveDIs] Enable direct-to-bitcode writing by default Follow on from #83251. This patch simply enables the behaviour by default in order to provide an easily revertible capstone. --- llvm/lib/IR/BasicBlock.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/IR/BasicBlock.cpp b/llvm/lib/IR/BasicBlock.cpp index 5f0780847938..3932576759d9 100644 --- a/llvm/lib/IR/BasicBlock.cpp +++ b/llvm/lib/IR/BasicBlock.cpp @@ -39,7 +39,7 @@ cl::opt bool WriteNewDbgInfoFormatToBitcode /*set default value in cl::init() below*/; cl::opt WriteNewDbgInfoFormatToBitcode2( "write-experimental-debuginfo-iterators-to-bitcode", cl::Hidden, - cl::location(WriteNewDbgInfoFormatToBitcode), cl::init(false)); + cl::location(WriteNewDbgInfoFormatToBitcode), cl::init(true)); DPMarker *BasicBlock::createMarker(Instruction *I) { assert(IsNewDbgInfoFormat && -- GitLab From 64f76dea9c95fd59e383e7550de35786781d9662 Mon Sep 17 00:00:00 2001 From: Orlando Cazalet-Hyams Date: Fri, 15 Mar 2024 13:40:44 +0000 Subject: [PATCH 008/514] [NFC] Fix comment in test from #83251 --- llvm/test/Bitcode/dbg-record-roundtrip.ll | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/llvm/test/Bitcode/dbg-record-roundtrip.ll b/llvm/test/Bitcode/dbg-record-roundtrip.ll index e7774a9d80d9..242c3819c860 100644 --- a/llvm/test/Bitcode/dbg-record-roundtrip.ll +++ b/llvm/test/Bitcode/dbg-record-roundtrip.ll @@ -1,7 +1,6 @@ ;; Roundtrip tests. ; RUN: llvm-as --write-experimental-debuginfo-iterators-to-bitcode=true %s -o - | llvm-dis | FileCheck %s ;; Check that verify-uselistorder passes regardless of input format. -;; NOTE: This test fails intermittently ; RUN: llvm-as %s --write-experimental-debuginfo-iterators-to-bitcode=true -o - | verify-uselistorder %s ; RUN: verify-uselistorder %s @@ -15,6 +14,9 @@ ;; Check that llvm-link doesn't explode if we give it different formats to ;; link. +;; NOTE: This test fails intermittently on linux if the llvm-as output is piped +;; into llvm-link in the RUN lines below, unless the verify-uselistorder RUN +;; lines above are removed. Write to a temporary file to avoid that weirdness. ; RUN: llvm-as %s --experimental-debuginfo-iterators=true --write-experimental-debuginfo-iterators-to-bitcode=true -o %t ; RUN: llvm-link %t %s --experimental-debuginfo-iterators=false -o /dev/null ; RUN: llvm-as %s --experimental-debuginfo-iterators=false -o %t -- GitLab From 0ae76a74985b3639ffb99d1dbb857068939547aa Mon Sep 17 00:00:00 2001 From: Orlando Cazalet-Hyams Date: Fri, 15 Mar 2024 14:01:14 +0000 Subject: [PATCH 009/514] [NFC] Fix incorrect RUN line in test from #83251 Note: This wasn't the cause of the strange behaviour mentioned in the NOTE comment in the test. --- llvm/test/Bitcode/dbg-record-roundtrip.ll | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/test/Bitcode/dbg-record-roundtrip.ll b/llvm/test/Bitcode/dbg-record-roundtrip.ll index 242c3819c860..d092e3822d26 100644 --- a/llvm/test/Bitcode/dbg-record-roundtrip.ll +++ b/llvm/test/Bitcode/dbg-record-roundtrip.ll @@ -1,7 +1,7 @@ ;; Roundtrip tests. ; RUN: llvm-as --write-experimental-debuginfo-iterators-to-bitcode=true %s -o - | llvm-dis | FileCheck %s ;; Check that verify-uselistorder passes regardless of input format. -; RUN: llvm-as %s --write-experimental-debuginfo-iterators-to-bitcode=true -o - | verify-uselistorder %s +; RUN: llvm-as %s --write-experimental-debuginfo-iterators-to-bitcode=true -o - | verify-uselistorder ; RUN: verify-uselistorder %s ;; Confirm we're producing RemoveDI records from various tools. -- GitLab From 0b9f19a9880eb786871194af116f223d2ad30c52 Mon Sep 17 00:00:00 2001 From: antoine moynault Date: Fri, 15 Mar 2024 15:07:40 +0100 Subject: [PATCH 010/514] Revert "[compiler-rt] Avoid generating coredumps when piped to a tool" (#85390) This reverts commit 27e5312a8bc8935f9c5620ff061c647d9fbcec85. This commit broke some bots: - clang-aarch64-sve-vla https://lab.llvm.org/buildbot/#/builders/197/builds/13609 - clang-aarch64-sve-vls https://lab.llvm.org/buildbot/#/builders/184/builds/10988 - clang-aarch64-lld-2stage https://lab.llvm.org/buildbot/#/builders/185/builds/6312 https://github.com/llvm/llvm-project/pull/83701 --- .../sanitizer_posix_libcdep.cpp | 19 +------------------ .../sanitizer_common/TestCases/corelimit.cpp | 7 +------ 2 files changed, 2 insertions(+), 24 deletions(-) diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cpp index 3605d0d666e3..ef1fc3549743 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cpp @@ -104,24 +104,7 @@ static void setlim(int res, rlim_t lim) { void DisableCoreDumperIfNecessary() { if (common_flags()->disable_coredump) { - rlimit rlim; - CHECK_EQ(0, getrlimit(RLIMIT_CORE, &rlim)); - // On Linux, if the kernel.core_pattern sysctl starts with a '|' (i.e. it - // is being piped to a coredump handler such as systemd-coredumpd), the - // kernel ignores RLIMIT_CORE (since we aren't creating a file in the file - // system) except for the magic value of 1, which disables coredumps when - // piping. 1 byte is too small for any kind of valid core dump, so it - // also disables coredumps if kernel.core_pattern creates files directly. - // While most piped coredump handlers do respect the crashing processes' - // RLIMIT_CORE, this is notable not the case for Debian's systemd-coredump - // due to a local patch that changes sysctl.d/50-coredump.conf to ignore - // the specified limit and instead use RLIM_INFINITY. - // - // The alternative to using RLIMIT_CORE=1 would be to use prctl() with the - // PR_SET_DUMPABLE flag, however that also prevents ptrace(), so makes it - // impossible to attach a debugger. - rlim.rlim_cur = Min(SANITIZER_LINUX ? 1 : 0, rlim.rlim_max); - CHECK_EQ(0, setrlimit(RLIMIT_CORE, &rlim)); + setlim(RLIMIT_CORE, 0); } } diff --git a/compiler-rt/test/sanitizer_common/TestCases/corelimit.cpp b/compiler-rt/test/sanitizer_common/TestCases/corelimit.cpp index fed2e1d89cbf..2378a4cfdced 100644 --- a/compiler-rt/test/sanitizer_common/TestCases/corelimit.cpp +++ b/compiler-rt/test/sanitizer_common/TestCases/corelimit.cpp @@ -10,12 +10,7 @@ int main() { getrlimit(RLIMIT_CORE, &lim_core); void *p; if (sizeof(p) == 8) { -#ifdef __linux__ - // See comments in DisableCoreDumperIfNecessary(). - assert(lim_core.rlim_cur == 1); -#else - assert(lim_core.rlim_cur == 0); -#endif + assert(0 == lim_core.rlim_cur); } return 0; } -- GitLab From f4676b6be6ee0d908c92d64936d17bd6fa3fbda8 Mon Sep 17 00:00:00 2001 From: Phoebe Wang Date: Fri, 15 Mar 2024 22:09:56 +0800 Subject: [PATCH 011/514] [X86] Add Support for X86 TLSDESC Relocations (#83136) --- clang/lib/Driver/ToolChains/CommonArgs.cpp | 3 +- clang/test/Driver/tls-dialect.c | 2 +- llvm/lib/Target/X86/X86ISelDAGToDAG.cpp | 16 +- llvm/lib/Target/X86/X86ISelLowering.cpp | 38 +++- llvm/lib/Target/X86/X86ISelLowering.h | 4 + llvm/lib/Target/X86/X86InstrCompiler.td | 10 ++ llvm/lib/Target/X86/X86InstrFragments.td | 3 + llvm/lib/Target/X86/X86MCInstLower.cpp | 33 +++- llvm/test/CodeGen/X86/tls-desc.ll | 199 +++++++++++++++++++++ 9 files changed, 289 insertions(+), 19 deletions(-) create mode 100644 llvm/test/CodeGen/X86/tls-desc.ll diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp index 100e71245394..83015b0cb81a 100644 --- a/clang/lib/Driver/ToolChains/CommonArgs.cpp +++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp @@ -740,7 +740,8 @@ bool tools::isTLSDESCEnabled(const ToolChain &TC, SupportedArgument = V == "desc" || V == "trad"; EnableTLSDESC = V == "desc"; } else if (Triple.isX86()) { - SupportedArgument = V == "gnu"; + SupportedArgument = V == "gnu" || V == "gnu2"; + EnableTLSDESC = V == "gnu2"; } else { Unsupported = true; } diff --git a/clang/test/Driver/tls-dialect.c b/clang/test/Driver/tls-dialect.c index f73915b28ec2..a808dd81531c 100644 --- a/clang/test/Driver/tls-dialect.c +++ b/clang/test/Driver/tls-dialect.c @@ -2,6 +2,7 @@ // RUN: %clang -### --target=riscv64-linux -mtls-dialect=trad %s 2>&1 | FileCheck --check-prefix=NODESC %s // RUN: %clang -### --target=riscv64-linux %s 2>&1 | FileCheck --check-prefix=NODESC %s // RUN: %clang -### --target=x86_64-linux -mtls-dialect=gnu %s 2>&1 | FileCheck --check-prefix=NODESC %s +// RUN: %clang -### --target=x86_64-linux -mtls-dialect=gnu2 %s 2>&1 | FileCheck --check-prefix=DESC %s /// Android supports TLSDESC by default on RISC-V /// TLSDESC is not on by default in Linux, even on RISC-V, and is covered above @@ -18,7 +19,6 @@ /// Unsupported argument // RUN: not %clang -### --target=riscv64-linux -mtls-dialect=gnu2 %s 2>&1 | FileCheck --check-prefix=UNSUPPORTED-ARG %s -// RUN: not %clang -### --target=x86_64-linux -mtls-dialect=gnu2 %s 2>&1 | FileCheck --check-prefix=UNSUPPORTED-ARG %s // DESC: "-cc1" {{.*}}"-enable-tlsdesc" // NODESC-NOT: "-enable-tlsdesc" diff --git a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp index 76c6c1645239..4e4241efd63d 100644 --- a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp +++ b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp @@ -3090,13 +3090,19 @@ bool X86DAGToDAGISel::selectLEAAddr(SDValue N, bool X86DAGToDAGISel::selectTLSADDRAddr(SDValue N, SDValue &Base, SDValue &Scale, SDValue &Index, SDValue &Disp, SDValue &Segment) { - assert(N.getOpcode() == ISD::TargetGlobalTLSAddress); - auto *GA = cast(N); + assert(N.getOpcode() == ISD::TargetGlobalTLSAddress || + N.getOpcode() == ISD::TargetExternalSymbol); X86ISelAddressMode AM; - AM.GV = GA->getGlobal(); - AM.Disp += GA->getOffset(); - AM.SymbolFlags = GA->getTargetFlags(); + if (auto *GA = dyn_cast(N)) { + AM.GV = GA->getGlobal(); + AM.Disp += GA->getOffset(); + AM.SymbolFlags = GA->getTargetFlags(); + } else { + auto *SA = cast(N); + AM.ES = SA->getSymbol(); + AM.SymbolFlags = SA->getTargetFlags(); + } if (Subtarget->is32Bit()) { AM.Scale = 1; diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index 2b5e3c0379a1..dbfcb3752ae8 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -18592,13 +18592,22 @@ GetTLSADDR(SelectionDAG &DAG, SDValue Chain, GlobalAddressSDNode *GA, MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); SDLoc dl(GA); - SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl, - GA->getValueType(0), - GA->getOffset(), - OperandFlags); + SDValue TGA; + bool UseTLSDESC = DAG.getTarget().useTLSDESC(); + if (LocalDynamic && UseTLSDESC) { + TGA = DAG.getTargetExternalSymbol("_TLS_MODULE_BASE_", PtrVT, OperandFlags); + auto UI = TGA->use_begin(); + // Reuse existing GetTLSADDR node if we can find it. + if (UI != TGA->use_end()) + return SDValue(*UI->use_begin()->use_begin(), 0); + } else { + TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl, GA->getValueType(0), + GA->getOffset(), OperandFlags); + } - X86ISD::NodeType CallType = LocalDynamic ? X86ISD::TLSBASEADDR - : X86ISD::TLSADDR; + X86ISD::NodeType CallType = UseTLSDESC ? X86ISD::TLSDESC + : LocalDynamic ? X86ISD::TLSBASEADDR + : X86ISD::TLSADDR; if (InGlue) { SDValue Ops[] = { Chain, TGA, *InGlue }; @@ -18613,7 +18622,19 @@ GetTLSADDR(SelectionDAG &DAG, SDValue Chain, GlobalAddressSDNode *GA, MFI.setHasCalls(true); SDValue Glue = Chain.getValue(1); - return DAG.getCopyFromReg(Chain, dl, ReturnReg, PtrVT, Glue); + SDValue Ret = DAG.getCopyFromReg(Chain, dl, ReturnReg, PtrVT, Glue); + + if (!UseTLSDESC) + return Ret; + + const X86Subtarget &Subtarget = DAG.getSubtarget(); + unsigned Seg = Subtarget.is64Bit() ? X86AS::FS : X86AS::GS; + + Value *Ptr = Constant::getNullValue(PointerType::get(*DAG.getContext(), Seg)); + SDValue Offset = + DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), DAG.getIntPtrConstant(0, dl), + MachinePointerInfo(Ptr)); + return DAG.getNode(ISD::ADD, dl, PtrVT, Ret, Offset); } // Lower ISD::GlobalTLSAddress using the "general dynamic" model, 32 bit @@ -33426,6 +33447,7 @@ const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const { NODE_NAME_CASE(TLSADDR) NODE_NAME_CASE(TLSBASEADDR) NODE_NAME_CASE(TLSCALL) + NODE_NAME_CASE(TLSDESC) NODE_NAME_CASE(EH_SJLJ_SETJMP) NODE_NAME_CASE(EH_SJLJ_LONGJMP) NODE_NAME_CASE(EH_SJLJ_SETUP_DISPATCH) @@ -36206,6 +36228,8 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, case X86::TLS_base_addr32: case X86::TLS_base_addr64: case X86::TLS_base_addrX32: + case X86::TLS_desc32: + case X86::TLS_desc64: return EmitLoweredTLSAddr(MI, BB); case X86::INDIRECT_THUNK_CALL32: case X86::INDIRECT_THUNK_CALL64: diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h index fe1943b57608..0a1e8ca44273 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.h +++ b/llvm/lib/Target/X86/X86ISelLowering.h @@ -295,6 +295,10 @@ namespace llvm { // thunk at the address from an earlier relocation. TLSCALL, + // Thread Local Storage. A descriptor containing pointer to + // code and to argument to get the TLS offset for the symbol. + TLSDESC, + // Exception Handling helpers. EH_RETURN, diff --git a/llvm/lib/Target/X86/X86InstrCompiler.td b/llvm/lib/Target/X86/X86InstrCompiler.td index f393f86e64aa..ce3b6af4cab4 100644 --- a/llvm/lib/Target/X86/X86InstrCompiler.td +++ b/llvm/lib/Target/X86/X86InstrCompiler.td @@ -507,6 +507,16 @@ def TLS_base_addrX32 : I<0, Pseudo, (outs), (ins i32mem:$sym), Requires<[In64BitMode, NotLP64]>; } +// TLSDESC only clobbers EAX and EFLAGS. ESP is marked as a use to prevent +// stack-pointer assignments that appear immediately before calls from +// potentially appearing dead. +let Defs = [EAX, EFLAGS], usesCustomInserter = 1, Uses = [RSP, SSP] in { + def TLS_desc32 : I<0, Pseudo, (outs), (ins i32mem:$sym), + "# TLS_desc32", [(X86tlsdesc tls32addr:$sym)]>; + def TLS_desc64 : I<0, Pseudo, (outs), (ins i64mem:$sym), + "# TLS_desc64", [(X86tlsdesc tls64addr:$sym)]>; +} + // Darwin TLS Support // For i386, the address of the thunk is passed on the stack, on return the // address of the variable is in %eax. %ecx is trashed during the function diff --git a/llvm/lib/Target/X86/X86InstrFragments.td b/llvm/lib/Target/X86/X86InstrFragments.td index adf527d72f5b..f14c7200af96 100644 --- a/llvm/lib/Target/X86/X86InstrFragments.td +++ b/llvm/lib/Target/X86/X86InstrFragments.td @@ -223,6 +223,9 @@ def X86tlsaddr : SDNode<"X86ISD::TLSADDR", SDT_X86TLSADDR, def X86tlsbaseaddr : SDNode<"X86ISD::TLSBASEADDR", SDT_X86TLSBASEADDR, [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue]>; +def X86tlsdesc : SDNode<"X86ISD::TLSDESC", SDT_X86TLSADDR, + [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue]>; + def X86ehret : SDNode<"X86ISD::EH_RETURN", SDT_X86EHRET, [SDNPHasChain]>; diff --git a/llvm/lib/Target/X86/X86MCInstLower.cpp b/llvm/lib/Target/X86/X86MCInstLower.cpp index 64d4d411e7b4..e2330ff34c17 100644 --- a/llvm/lib/Target/X86/X86MCInstLower.cpp +++ b/llvm/lib/Target/X86/X86MCInstLower.cpp @@ -519,10 +519,8 @@ void X86MCInstLower::Lower(const MachineInstr *MI, MCInst &OutMI) const { void X86AsmPrinter::LowerTlsAddr(X86MCInstLower &MCInstLowering, const MachineInstr &MI) { NoAutoPaddingScope NoPadScope(*OutStreamer); - bool Is64Bits = MI.getOpcode() != X86::TLS_addr32 && - MI.getOpcode() != X86::TLS_base_addr32; - bool Is64BitsLP64 = MI.getOpcode() == X86::TLS_addr64 || - MI.getOpcode() == X86::TLS_base_addr64; + bool Is64Bits = getSubtarget().is64Bit(); + bool Is64BitsLP64 = getSubtarget().isTarget64BitLP64(); MCContext &Ctx = OutStreamer->getContext(); MCSymbolRefExpr::VariantKind SRVK; @@ -539,6 +537,10 @@ void X86AsmPrinter::LowerTlsAddr(X86MCInstLower &MCInstLowering, case X86::TLS_base_addrX32: SRVK = MCSymbolRefExpr::VK_TLSLD; break; + case X86::TLS_desc32: + case X86::TLS_desc64: + SRVK = MCSymbolRefExpr::VK_TLSDESC; + break; default: llvm_unreachable("unexpected opcode"); } @@ -554,7 +556,26 @@ void X86AsmPrinter::LowerTlsAddr(X86MCInstLower &MCInstLowering, bool UseGot = MMI->getModule()->getRtLibUseGOT() && Ctx.getTargetOptions()->X86RelaxRelocations; - if (Is64Bits) { + if (SRVK == MCSymbolRefExpr::VK_TLSDESC) { + const MCSymbolRefExpr *Expr = MCSymbolRefExpr::create( + MCInstLowering.GetSymbolFromOperand(MI.getOperand(3)), + MCSymbolRefExpr::VK_TLSCALL, Ctx); + EmitAndCountInstruction( + MCInstBuilder(Is64BitsLP64 ? X86::LEA64r : X86::LEA32r) + .addReg(Is64BitsLP64 ? X86::RAX : X86::EAX) + .addReg(Is64Bits ? X86::RIP : X86::EBX) + .addImm(1) + .addReg(0) + .addExpr(Sym) + .addReg(0)); + EmitAndCountInstruction( + MCInstBuilder(Is64Bits ? X86::CALL64m : X86::CALL32m) + .addReg(Is64BitsLP64 ? X86::RAX : X86::EAX) + .addImm(1) + .addReg(0) + .addExpr(Expr) + .addReg(0)); + } else if (Is64Bits) { bool NeedsPadding = SRVK == MCSymbolRefExpr::VK_TLSGD; if (NeedsPadding && Is64BitsLP64) EmitAndCountInstruction(MCInstBuilder(X86::DATA16_PREFIX)); @@ -2164,6 +2185,8 @@ void X86AsmPrinter::emitInstruction(const MachineInstr *MI) { case X86::TLS_base_addr32: case X86::TLS_base_addr64: case X86::TLS_base_addrX32: + case X86::TLS_desc32: + case X86::TLS_desc64: return LowerTlsAddr(MCInstLowering, *MI); case X86::MOVPC32r: { diff --git a/llvm/test/CodeGen/X86/tls-desc.ll b/llvm/test/CodeGen/X86/tls-desc.ll new file mode 100644 index 000000000000..c73986e69e79 --- /dev/null +++ b/llvm/test/CodeGen/X86/tls-desc.ll @@ -0,0 +1,199 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc < %s -mtriple=i686 --relocation-model=pic -enable-tlsdesc | FileCheck %s --check-prefix=X86 +; RUN: llc < %s -mtriple=x86_64-pc-linux-gnux32 --relocation-model=pic -enable-tlsdesc | FileCheck %s --check-prefix=X32 +; RUN: llc < %s -mtriple=x86_64 --relocation-model=pic -enable-tlsdesc | FileCheck %s --check-prefix=X64 + +@x = thread_local global i32 0, align 4 +@y = internal thread_local global i32 1, align 4 +@z = external hidden thread_local global i32, align 4 + +define ptr @f1() nounwind { +; X86-LABEL: f1: +; X86: # %bb.0: +; X86-NEXT: pushl %ebp +; X86-NEXT: pushl %ebx +; X86-NEXT: pushl %edi +; X86-NEXT: pushl %esi +; X86-NEXT: pushl %eax +; X86-NEXT: calll .L0$pb +; X86-NEXT: .L0$pb: +; X86-NEXT: popl %ebx +; X86-NEXT: .Ltmp0: +; X86-NEXT: addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp0-.L0$pb), %ebx +; X86-NEXT: #APP +; X86-NEXT: #NO_APP +; X86-NEXT: movl %eax, (%esp) # 4-byte Spill +; X86-NEXT: leal x@tlsdesc(%ebx), %eax +; X86-NEXT: calll *x@tlscall(%eax) +; X86-NEXT: addl %gs:0, %eax +; X86-NEXT: movl (%esp), %ebx # 4-byte Reload +; X86-NEXT: #APP +; X86-NEXT: #NO_APP +; X86-NEXT: addl $4, %esp +; X86-NEXT: popl %esi +; X86-NEXT: popl %edi +; X86-NEXT: popl %ebx +; X86-NEXT: popl %ebp +; X86-NEXT: retl +; +; X32-LABEL: f1: +; X32: # %bb.0: +; X32-NEXT: pushq %rax +; X32-NEXT: #APP +; X32-NEXT: #NO_APP +; X32-NEXT: leal x@tlsdesc(%rip), %eax +; X32-NEXT: callq *x@tlscall(%eax) +; X32-NEXT: # kill: def $eax killed $eax def $rax +; X32-NEXT: addl %fs:0, %eax +; X32-NEXT: #APP +; X32-NEXT: #NO_APP +; X32-NEXT: popq %rcx +; X32-NEXT: retq +; +; X64-LABEL: f1: +; X64: # %bb.0: +; X64-NEXT: pushq %rax +; X64-NEXT: #APP +; X64-NEXT: #NO_APP +; X64-NEXT: leaq x@tlsdesc(%rip), %rax +; X64-NEXT: callq *x@tlscall(%rax) +; X64-NEXT: addq %fs:0, %rax +; X64-NEXT: #APP +; X64-NEXT: #NO_APP +; X64-NEXT: popq %rcx +; X64-NEXT: retq + %a = call { i32, i32, i32, i32, i32, i32 } asm sideeffect "", "=r,=r,=r,=r,=r,=r,~{dirflag},~{fpsr},~{flags}"() + %b = call ptr @llvm.threadlocal.address.p0(ptr @x) + %a.0 = extractvalue { i32, i32, i32, i32, i32, i32 } %a, 0 + %a.1 = extractvalue { i32, i32, i32, i32, i32, i32 } %a, 1 + %a.2 = extractvalue { i32, i32, i32, i32, i32, i32 } %a, 2 + %a.3 = extractvalue { i32, i32, i32, i32, i32, i32 } %a, 3 + %a.4 = extractvalue { i32, i32, i32, i32, i32, i32 } %a, 4 + %a.5 = extractvalue { i32, i32, i32, i32, i32, i32 } %a, 5 + call void asm sideeffect "", "r,r,r,r,r,r,~{dirflag},~{fpsr},~{flags}"(i32 %a.0, i32 %a.1, i32 %a.2, i32 %a.3, i32 %a.4, i32 %a.5) + ret ptr %b +} + +define i32 @f2() nounwind { +; X86-LABEL: f2: +; X86: # %bb.0: +; X86-NEXT: pushl %ebx +; X86-NEXT: calll .L1$pb +; X86-NEXT: .L1$pb: +; X86-NEXT: popl %ebx +; X86-NEXT: .Ltmp1: +; X86-NEXT: addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp1-.L1$pb), %ebx +; X86-NEXT: movl %gs:0, %ecx +; X86-NEXT: leal x@tlsdesc(%ebx), %eax +; X86-NEXT: calll *x@tlscall(%eax) +; X86-NEXT: movl (%eax,%ecx), %eax +; X86-NEXT: popl %ebx +; X86-NEXT: retl +; +; X32-LABEL: f2: +; X32: # %bb.0: +; X32-NEXT: pushq %rax +; X32-NEXT: movl %fs:0, %ecx +; X32-NEXT: leal x@tlsdesc(%rip), %eax +; X32-NEXT: callq *x@tlscall(%eax) +; X32-NEXT: movl (%eax,%ecx), %eax +; X32-NEXT: popq %rcx +; X32-NEXT: retq +; +; X64-LABEL: f2: +; X64: # %bb.0: +; X64-NEXT: pushq %rax +; X64-NEXT: movq %fs:0, %rcx +; X64-NEXT: leaq x@tlsdesc(%rip), %rax +; X64-NEXT: callq *x@tlscall(%rax) +; X64-NEXT: movl (%rax,%rcx), %eax +; X64-NEXT: popq %rcx +; X64-NEXT: retq + %1 = tail call ptr @llvm.threadlocal.address.p0(ptr @x) + %2 = load i32, ptr %1 + ret i32 %2 +} + +define ptr @f3() nounwind { +; X86-LABEL: f3: +; X86: # %bb.0: +; X86-NEXT: pushl %ebx +; X86-NEXT: calll .L2$pb +; X86-NEXT: .L2$pb: +; X86-NEXT: popl %ebx +; X86-NEXT: .Ltmp2: +; X86-NEXT: addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp2-.L2$pb), %ebx +; X86-NEXT: leal x@tlsdesc(%ebx), %eax +; X86-NEXT: calll *x@tlscall(%eax) +; X86-NEXT: addl %gs:0, %eax +; X86-NEXT: popl %ebx +; X86-NEXT: retl +; +; X32-LABEL: f3: +; X32: # %bb.0: +; X32-NEXT: pushq %rax +; X32-NEXT: leal x@tlsdesc(%rip), %eax +; X32-NEXT: callq *x@tlscall(%eax) +; X32-NEXT: # kill: def $eax killed $eax def $rax +; X32-NEXT: addl %fs:0, %eax +; X32-NEXT: popq %rcx +; X32-NEXT: retq +; +; X64-LABEL: f3: +; X64: # %bb.0: +; X64-NEXT: pushq %rax +; X64-NEXT: leaq x@tlsdesc(%rip), %rax +; X64-NEXT: callq *x@tlscall(%rax) +; X64-NEXT: addq %fs:0, %rax +; X64-NEXT: popq %rcx +; X64-NEXT: retq + %1 = tail call ptr @llvm.threadlocal.address.p0(ptr @x) + ret ptr %1 +} + +define i32 @f4() nounwind { +; X86-LABEL: f4: +; X86: # %bb.0: +; X86-NEXT: pushl %ebx +; X86-NEXT: calll .L3$pb +; X86-NEXT: .L3$pb: +; X86-NEXT: popl %ebx +; X86-NEXT: .Ltmp3: +; X86-NEXT: addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp3-.L3$pb), %ebx +; X86-NEXT: movl %gs:0, %edx +; X86-NEXT: leal _TLS_MODULE_BASE_@tlsdesc(%ebx), %eax +; X86-NEXT: calll *_TLS_MODULE_BASE_@tlscall(%eax) +; X86-NEXT: movl y@DTPOFF(%eax,%edx), %ecx +; X86-NEXT: addl z@DTPOFF(%eax,%edx), %ecx +; X86-NEXT: movl %ecx, %eax +; X86-NEXT: popl %ebx +; X86-NEXT: retl +; +; X32-LABEL: f4: +; X32: # %bb.0: +; X32-NEXT: pushq %rax +; X32-NEXT: movl %fs:0, %edx +; X32-NEXT: leal _TLS_MODULE_BASE_@tlsdesc(%rip), %eax +; X32-NEXT: callq *_TLS_MODULE_BASE_@tlscall(%eax) +; X32-NEXT: movl y@DTPOFF(%eax,%edx), %ecx +; X32-NEXT: addl z@DTPOFF(%eax,%edx), %ecx +; X32-NEXT: movl %ecx, %eax +; X32-NEXT: popq %rcx +; X32-NEXT: retq +; +; X64-LABEL: f4: +; X64: # %bb.0: +; X64-NEXT: pushq %rax +; X64-NEXT: movq %fs:0, %rdx +; X64-NEXT: leaq _TLS_MODULE_BASE_@tlsdesc(%rip), %rax +; X64-NEXT: callq *_TLS_MODULE_BASE_@tlscall(%rax) +; X64-NEXT: movl y@DTPOFF(%rax,%rdx), %ecx +; X64-NEXT: addl z@DTPOFF(%rax,%rdx), %ecx +; X64-NEXT: movl %ecx, %eax +; X64-NEXT: popq %rcx +; X64-NEXT: retq + %1 = load i32, ptr @y, align 4 + %2 = load i32, ptr @z, align 4 + %3 = add nsw i32 %1, %2 + ret i32 %3 +} -- GitLab From 092999e70b349ac521cab2648152ababeb12873f Mon Sep 17 00:00:00 2001 From: Jay Foad Date: Fri, 15 Mar 2024 14:08:43 +0000 Subject: [PATCH 012/514] [AMDGPU] Update checks in new test after #85370 --- llvm/test/CodeGen/AMDGPU/itofp.i128.ll | 2186 +++++++++++------------- 1 file changed, 1001 insertions(+), 1185 deletions(-) diff --git a/llvm/test/CodeGen/AMDGPU/itofp.i128.ll b/llvm/test/CodeGen/AMDGPU/itofp.i128.ll index e4e8d52addf8..bfeb214c5af8 100644 --- a/llvm/test/CodeGen/AMDGPU/itofp.i128.ll +++ b/llvm/test/CodeGen/AMDGPU/itofp.i128.ll @@ -11,7 +11,7 @@ define float @sitofp_i128_to_f32(i128 %x) { ; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; SDAG-NEXT: v_mov_b32_e32 v4, 0 ; SDAG-NEXT: s_and_saveexec_b64 s[6:7], vcc -; SDAG-NEXT: s_cbranch_execz .LBB0_16 +; SDAG-NEXT: s_cbranch_execz .LBB0_14 ; SDAG-NEXT: ; %bb.1: ; %itofp-if-end ; SDAG-NEXT: v_ashrrev_i32_e32 v5, 31, v3 ; SDAG-NEXT: v_xor_b32_e32 v0, v5, v0 @@ -32,112 +32,100 @@ define float @sitofp_i128_to_f32(i128 %x) { ; SDAG-NEXT: v_min_u32_e32 v6, v6, v7 ; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; SDAG-NEXT: v_add_u32_e32 v6, 64, v6 -; SDAG-NEXT: v_cndmask_b32_e32 v9, v6, v2, vcc -; SDAG-NEXT: v_sub_u32_e32 v2, 0x7f, v9 -; SDAG-NEXT: v_cmp_gt_i32_e32 vcc, 25, v2 -; SDAG-NEXT: ; implicit-def: $vgpr6 +; SDAG-NEXT: v_cndmask_b32_e32 v7, v6, v2, vcc +; SDAG-NEXT: v_sub_u32_e32 v6, 0x80, v7 +; SDAG-NEXT: v_sub_u32_e32 v2, 0x7f, v7 +; SDAG-NEXT: v_cmp_gt_i32_e32 vcc, 25, v6 +; SDAG-NEXT: ; implicit-def: $vgpr8 ; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc ; SDAG-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; SDAG-NEXT: ; %bb.2: ; %itofp-if-else -; SDAG-NEXT: v_add_u32_e32 v4, 0xffffff98, v9 +; SDAG-NEXT: v_add_u32_e32 v4, 0xffffff98, v7 ; SDAG-NEXT: v_lshlrev_b64 v[0:1], v4, v[0:1] ; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v4 -; SDAG-NEXT: v_cndmask_b32_e32 v6, 0, v0, vcc +; SDAG-NEXT: v_cndmask_b32_e32 v8, 0, v0, vcc +; SDAG-NEXT: ; implicit-def: $vgpr6 ; SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 -; SDAG-NEXT: ; implicit-def: $vgpr9 +; SDAG-NEXT: ; implicit-def: $vgpr7 ; SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5 -; SDAG-NEXT: ; %bb.3: ; %Flow6 +; SDAG-NEXT: ; %bb.3: ; %Flow3 ; SDAG-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] -; SDAG-NEXT: s_cbranch_execz .LBB0_15 +; SDAG-NEXT: s_cbranch_execz .LBB0_13 ; SDAG-NEXT: ; %bb.4: ; %NodeBlock -; SDAG-NEXT: v_sub_u32_e32 v8, 0x80, v9 -; SDAG-NEXT: v_cmp_lt_i32_e32 vcc, 25, v8 -; SDAG-NEXT: s_mov_b64 s[10:11], 0 -; SDAG-NEXT: s_mov_b64 s[4:5], 0 +; SDAG-NEXT: v_cmp_lt_i32_e32 vcc, 25, v6 +; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc +; SDAG-NEXT: s_xor_b64 s[10:11], exec, s[4:5] +; SDAG-NEXT: s_cbranch_execz .LBB0_8 +; SDAG-NEXT: ; %bb.5: ; %LeafBlock +; SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 26, v6 ; SDAG-NEXT: s_and_saveexec_b64 s[12:13], vcc -; SDAG-NEXT: s_xor_b64 s[12:13], exec, s[12:13] -; SDAG-NEXT: ; %bb.5: ; %LeafBlock1 -; SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 26, v8 -; SDAG-NEXT: s_and_b64 s[4:5], vcc, exec -; SDAG-NEXT: ; %bb.6: ; %Flow3 -; SDAG-NEXT: s_andn2_saveexec_b64 s[12:13], s[12:13] -; SDAG-NEXT: ; %bb.7: ; %LeafBlock -; SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 25, v8 -; SDAG-NEXT: s_andn2_b64 s[4:5], s[4:5], exec -; SDAG-NEXT: s_and_b64 s[14:15], vcc, exec -; SDAG-NEXT: s_mov_b64 s[10:11], exec -; SDAG-NEXT: s_or_b64 s[4:5], s[4:5], s[14:15] -; SDAG-NEXT: ; %bb.8: ; %Flow4 -; SDAG-NEXT: s_or_b64 exec, exec, s[12:13] -; SDAG-NEXT: v_mov_b32_e32 v7, v1 -; SDAG-NEXT: v_mov_b32_e32 v6, v0 -; SDAG-NEXT: s_and_saveexec_b64 s[12:13], s[4:5] -; SDAG-NEXT: s_xor_b64 s[12:13], exec, s[12:13] -; SDAG-NEXT: s_cbranch_execz .LBB0_10 -; SDAG-NEXT: ; %bb.9: ; %itofp-sw-default -; SDAG-NEXT: v_sub_u32_e32 v12, 0x66, v9 +; SDAG-NEXT: s_cbranch_execz .LBB0_7 +; SDAG-NEXT: ; %bb.6: ; %itofp-sw-default +; SDAG-NEXT: v_sub_u32_e32 v12, 0x66, v7 ; SDAG-NEXT: v_sub_u32_e32 v10, 64, v12 -; SDAG-NEXT: v_lshrrev_b64 v[6:7], v12, v[0:1] +; SDAG-NEXT: v_lshrrev_b64 v[8:9], v12, v[0:1] ; SDAG-NEXT: v_lshlrev_b64 v[10:11], v10, v[4:5] -; SDAG-NEXT: v_sub_u32_e32 v13, 38, v9 -; SDAG-NEXT: v_or_b32_e32 v11, v7, v11 -; SDAG-NEXT: v_or_b32_e32 v10, v6, v10 -; SDAG-NEXT: v_lshrrev_b64 v[6:7], v13, v[4:5] +; SDAG-NEXT: v_sub_u32_e32 v13, 38, v7 +; SDAG-NEXT: v_or_b32_e32 v11, v9, v11 +; SDAG-NEXT: v_or_b32_e32 v10, v8, v10 +; SDAG-NEXT: v_lshrrev_b64 v[8:9], v13, v[4:5] ; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v12 -; SDAG-NEXT: v_add_u32_e32 v14, 26, v9 -; SDAG-NEXT: v_cndmask_b32_e32 v7, v7, v11, vcc +; SDAG-NEXT: v_add_u32_e32 v14, 26, v7 +; SDAG-NEXT: v_cndmask_b32_e32 v9, v9, v11, vcc ; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v12 -; SDAG-NEXT: v_cndmask_b32_e32 v6, v6, v10, vcc +; SDAG-NEXT: v_cndmask_b32_e32 v8, v8, v10, vcc ; SDAG-NEXT: v_lshrrev_b64 v[10:11], v13, v[0:1] ; SDAG-NEXT: v_lshlrev_b64 v[12:13], v14, v[4:5] -; SDAG-NEXT: v_subrev_u32_e32 v9, 38, v9 -; SDAG-NEXT: v_cndmask_b32_e64 v15, v6, v0, s[4:5] -; SDAG-NEXT: v_or_b32_e32 v6, v13, v11 -; SDAG-NEXT: v_or_b32_e32 v11, v12, v10 -; SDAG-NEXT: v_lshlrev_b64 v[9:10], v9, v[0:1] +; SDAG-NEXT: v_subrev_u32_e32 v7, 38, v7 +; SDAG-NEXT: v_cndmask_b32_e64 v15, v8, v0, s[4:5] +; SDAG-NEXT: v_lshlrev_b64 v[7:8], v7, v[0:1] +; SDAG-NEXT: v_cndmask_b32_e64 v9, v9, v1, s[4:5] +; SDAG-NEXT: v_or_b32_e32 v11, v13, v11 +; SDAG-NEXT: v_or_b32_e32 v10, v12, v10 ; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v14 -; SDAG-NEXT: v_cndmask_b32_e64 v7, v7, v1, s[4:5] -; SDAG-NEXT: v_cndmask_b32_e32 v6, v10, v6, vcc +; SDAG-NEXT: v_lshlrev_b64 v[0:1], v14, v[0:1] +; SDAG-NEXT: v_cndmask_b32_e32 v8, v8, v11, vcc ; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v14 -; SDAG-NEXT: v_cndmask_b32_e64 v10, v6, v5, s[4:5] -; SDAG-NEXT: v_lshlrev_b64 v[5:6], v14, v[0:1] -; SDAG-NEXT: v_cndmask_b32_e32 v9, v9, v11, vcc -; SDAG-NEXT: v_cndmask_b32_e64 v4, v9, v4, s[4:5] -; SDAG-NEXT: v_cndmask_b32_e32 v6, 0, v6, vcc -; SDAG-NEXT: v_cndmask_b32_e32 v9, 0, v5, vcc -; SDAG-NEXT: v_or_b32_e32 v5, v6, v10 -; SDAG-NEXT: v_or_b32_e32 v4, v9, v4 -; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] -; SDAG-NEXT: s_andn2_b64 s[10:11], s[10:11], exec -; SDAG-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc -; SDAG-NEXT: v_or_b32_e32 v6, v15, v4 -; SDAG-NEXT: .LBB0_10: ; %Flow5 +; SDAG-NEXT: v_cndmask_b32_e32 v7, v7, v10, vcc +; SDAG-NEXT: v_cndmask_b32_e64 v5, v8, v5, s[4:5] +; SDAG-NEXT: v_cndmask_b32_e64 v4, v7, v4, s[4:5] +; SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc +; SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc +; SDAG-NEXT: v_or_b32_e32 v1, v1, v5 +; SDAG-NEXT: v_or_b32_e32 v0, v0, v4 +; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] +; SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc +; SDAG-NEXT: v_or_b32_e32 v8, v15, v0 +; SDAG-NEXT: v_mov_b32_e32 v0, v8 +; SDAG-NEXT: v_mov_b32_e32 v1, v9 +; SDAG-NEXT: .LBB0_7: ; %Flow1 ; SDAG-NEXT: s_or_b64 exec, exec, s[12:13] -; SDAG-NEXT: s_and_saveexec_b64 s[4:5], s[10:11] -; SDAG-NEXT: ; %bb.11: ; %itofp-sw-bb -; SDAG-NEXT: v_lshlrev_b64 v[6:7], 1, v[0:1] -; SDAG-NEXT: ; %bb.12: ; %itofp-sw-epilog +; SDAG-NEXT: .LBB0_8: ; %Flow2 +; SDAG-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11] +; SDAG-NEXT: ; %bb.9: ; %itofp-sw-bb +; SDAG-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1] +; SDAG-NEXT: ; %bb.10: ; %itofp-sw-epilog ; SDAG-NEXT: s_or_b64 exec, exec, s[4:5] -; SDAG-NEXT: v_lshrrev_b32_e32 v0, 2, v6 -; SDAG-NEXT: v_and_or_b32 v0, v0, 1, v6 +; SDAG-NEXT: v_lshrrev_b32_e32 v4, 2, v0 +; SDAG-NEXT: v_and_or_b32 v0, v4, 1, v0 ; SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 1, v0 -; SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v7, vcc +; SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc ; SDAG-NEXT: v_and_b32_e32 v4, 0x4000000, v0 ; SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4 -; SDAG-NEXT: v_alignbit_b32 v6, v1, v0, 2 +; SDAG-NEXT: v_alignbit_b32 v8, v1, v0, 2 ; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc -; SDAG-NEXT: ; %bb.13: ; %itofp-if-then20 -; SDAG-NEXT: v_alignbit_b32 v6, v1, v0, 3 -; SDAG-NEXT: v_mov_b32_e32 v2, v8 -; SDAG-NEXT: ; %bb.14: ; %Flow +; SDAG-NEXT: ; %bb.11: ; %itofp-if-then20 +; SDAG-NEXT: v_alignbit_b32 v8, v1, v0, 3 +; SDAG-NEXT: v_mov_b32_e32 v2, v6 +; SDAG-NEXT: ; %bb.12: ; %Flow ; SDAG-NEXT: s_or_b64 exec, exec, s[4:5] -; SDAG-NEXT: .LBB0_15: ; %Flow7 +; SDAG-NEXT: .LBB0_13: ; %Flow4 ; SDAG-NEXT: s_or_b64 exec, exec, s[8:9] ; SDAG-NEXT: v_and_b32_e32 v0, 0x80000000, v3 ; SDAG-NEXT: v_lshl_add_u32 v1, v2, 23, 1.0 -; SDAG-NEXT: v_and_b32_e32 v2, 0x7fffff, v6 +; SDAG-NEXT: v_and_b32_e32 v2, 0x7fffff, v8 ; SDAG-NEXT: v_or3_b32 v4, v2, v0, v1 -; SDAG-NEXT: .LBB0_16: ; %Flow8 +; SDAG-NEXT: .LBB0_14: ; %Flow5 ; SDAG-NEXT: s_or_b64 exec, exec, s[6:7] ; SDAG-NEXT: v_mov_b32_e32 v0, v4 ; SDAG-NEXT: s_setpc_b64 s[30:31] @@ -151,144 +139,126 @@ define float @sitofp_i128_to_f32(i128 %x) { ; GISEL-NEXT: s_mov_b32 s4, 0 ; GISEL-NEXT: v_mov_b32_e32 v4, s4 ; GISEL-NEXT: s_and_saveexec_b64 s[6:7], vcc -; GISEL-NEXT: s_cbranch_execz .LBB0_16 +; GISEL-NEXT: s_cbranch_execz .LBB0_14 ; GISEL-NEXT: ; %bb.1: ; %itofp-if-end -; GISEL-NEXT: v_ashrrev_i32_e32 v8, 31, v3 -; GISEL-NEXT: v_xor_b32_e32 v0, v8, v0 -; GISEL-NEXT: v_xor_b32_e32 v1, v8, v1 -; GISEL-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v8 -; GISEL-NEXT: v_xor_b32_e32 v2, v8, v2 -; GISEL-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v8, vcc -; GISEL-NEXT: v_xor_b32_e32 v3, v8, v3 -; GISEL-NEXT: v_subb_co_u32_e32 v6, vcc, v2, v8, vcc -; GISEL-NEXT: v_subb_co_u32_e32 v7, vcc, v3, v8, vcc -; GISEL-NEXT: v_ffbh_u32_e32 v3, v0 -; GISEL-NEXT: v_ffbh_u32_e32 v2, v1 -; GISEL-NEXT: v_add_u32_e32 v3, 32, v3 -; GISEL-NEXT: v_ffbh_u32_e32 v4, v6 -; GISEL-NEXT: v_min_u32_e32 v2, v2, v3 -; GISEL-NEXT: v_ffbh_u32_e32 v3, v7 -; GISEL-NEXT: v_add_u32_e32 v4, 32, v4 -; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[6:7] -; GISEL-NEXT: v_add_u32_e32 v2, 64, v2 -; GISEL-NEXT: v_min_u32_e32 v3, v3, v4 -; GISEL-NEXT: v_cndmask_b32_e32 v11, v3, v2, vcc -; GISEL-NEXT: v_sub_u32_e32 v9, 0x7f, v11 -; GISEL-NEXT: v_cmp_ge_i32_e32 vcc, 24, v9 -; GISEL-NEXT: ; implicit-def: $vgpr2 +; GISEL-NEXT: v_ashrrev_i32_e32 v6, 31, v3 +; GISEL-NEXT: v_xor_b32_e32 v0, v6, v0 +; GISEL-NEXT: v_xor_b32_e32 v1, v6, v1 +; GISEL-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v6 +; GISEL-NEXT: v_xor_b32_e32 v2, v6, v2 +; GISEL-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v6, vcc +; GISEL-NEXT: v_xor_b32_e32 v3, v6, v3 +; GISEL-NEXT: v_subb_co_u32_e32 v2, vcc, v2, v6, vcc +; GISEL-NEXT: v_ffbh_u32_e32 v5, v0 +; GISEL-NEXT: v_subb_co_u32_e32 v3, vcc, v3, v6, vcc +; GISEL-NEXT: v_ffbh_u32_e32 v4, v1 +; GISEL-NEXT: v_add_u32_e32 v5, 32, v5 +; GISEL-NEXT: v_ffbh_u32_e32 v7, v2 +; GISEL-NEXT: v_min_u32_e32 v4, v4, v5 +; GISEL-NEXT: v_ffbh_u32_e32 v5, v3 +; GISEL-NEXT: v_add_u32_e32 v7, 32, v7 +; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3] +; GISEL-NEXT: v_add_u32_e32 v4, 64, v4 +; GISEL-NEXT: v_min_u32_e32 v5, v5, v7 +; GISEL-NEXT: v_cndmask_b32_e32 v5, v5, v4, vcc +; GISEL-NEXT: v_sub_u32_e32 v8, 0x80, v5 +; GISEL-NEXT: v_sub_u32_e32 v7, 0x7f, v5 +; GISEL-NEXT: v_cmp_ge_i32_e32 vcc, 24, v8 +; GISEL-NEXT: ; implicit-def: $vgpr4 ; GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GISEL-NEXT: ; %bb.2: ; %itofp-if-else -; GISEL-NEXT: v_add_u32_e32 v2, 0xffffff98, v11 +; GISEL-NEXT: v_add_u32_e32 v2, 0xffffff98, v5 ; GISEL-NEXT: v_lshlrev_b64 v[0:1], v2, v[0:1] ; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v2 -; GISEL-NEXT: v_cndmask_b32_e32 v2, 0, v0, vcc +; GISEL-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc +; GISEL-NEXT: ; implicit-def: $vgpr8 ; GISEL-NEXT: ; implicit-def: $vgpr0 -; GISEL-NEXT: ; implicit-def: $vgpr11 -; GISEL-NEXT: ; implicit-def: $vgpr6 -; GISEL-NEXT: ; %bb.3: ; %Flow6 +; GISEL-NEXT: ; implicit-def: $vgpr5 +; GISEL-NEXT: ; implicit-def: $vgpr2 +; GISEL-NEXT: ; %bb.3: ; %Flow3 ; GISEL-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] -; GISEL-NEXT: s_cbranch_execz .LBB0_15 +; GISEL-NEXT: s_cbranch_execz .LBB0_13 ; GISEL-NEXT: ; %bb.4: ; %NodeBlock -; GISEL-NEXT: v_sub_u32_e32 v10, 0x80, v11 -; GISEL-NEXT: v_cmp_le_i32_e32 vcc, 26, v10 -; GISEL-NEXT: s_mov_b64 s[10:11], 0 -; GISEL-NEXT: s_mov_b64 s[4:5], 0 +; GISEL-NEXT: v_cmp_le_i32_e32 vcc, 26, v8 +; GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GISEL-NEXT: s_xor_b64 s[10:11], exec, s[4:5] +; GISEL-NEXT: s_cbranch_execz .LBB0_8 +; GISEL-NEXT: ; %bb.5: ; %LeafBlock +; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 26, v8 ; GISEL-NEXT: s_and_saveexec_b64 s[12:13], vcc -; GISEL-NEXT: s_xor_b64 s[12:13], exec, s[12:13] -; GISEL-NEXT: ; %bb.5: ; %LeafBlock1 -; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 26, v10 -; GISEL-NEXT: s_andn2_b64 s[4:5], 0, exec -; GISEL-NEXT: s_and_b64 s[14:15], exec, vcc -; GISEL-NEXT: s_or_b64 s[4:5], s[4:5], s[14:15] -; GISEL-NEXT: ; %bb.6: ; %Flow3 -; GISEL-NEXT: s_andn2_saveexec_b64 s[12:13], s[12:13] -; GISEL-NEXT: ; %bb.7: ; %LeafBlock -; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 25, v10 -; GISEL-NEXT: s_andn2_b64 s[10:11], 0, exec -; GISEL-NEXT: s_and_b64 s[14:15], exec, -1 -; GISEL-NEXT: s_or_b64 s[10:11], s[10:11], s[14:15] -; GISEL-NEXT: s_andn2_b64 s[4:5], s[4:5], exec -; GISEL-NEXT: s_and_b64 s[14:15], exec, vcc -; GISEL-NEXT: s_or_b64 s[4:5], s[4:5], s[14:15] -; GISEL-NEXT: ; %bb.8: ; %Flow4 -; GISEL-NEXT: s_or_b64 exec, exec, s[12:13] -; GISEL-NEXT: v_mov_b32_e32 v5, v3 -; GISEL-NEXT: v_mov_b32_e32 v4, v2 -; GISEL-NEXT: v_mov_b32_e32 v3, v1 -; GISEL-NEXT: v_mov_b32_e32 v2, v0 -; GISEL-NEXT: s_and_saveexec_b64 s[12:13], s[4:5] -; GISEL-NEXT: s_xor_b64 s[12:13], exec, s[12:13] -; GISEL-NEXT: s_cbranch_execz .LBB0_10 -; GISEL-NEXT: ; %bb.9: ; %itofp-sw-default -; GISEL-NEXT: v_sub_u32_e32 v12, 0x66, v11 -; GISEL-NEXT: v_sub_u32_e32 v4, 64, v12 -; GISEL-NEXT: v_lshrrev_b64 v[2:3], v12, v[0:1] -; GISEL-NEXT: v_lshlrev_b64 v[4:5], v4, v[6:7] -; GISEL-NEXT: v_subrev_u32_e32 v13, 64, v12 -; GISEL-NEXT: v_or_b32_e32 v4, v2, v4 -; GISEL-NEXT: v_or_b32_e32 v5, v3, v5 -; GISEL-NEXT: v_lshrrev_b64 v[2:3], v13, v[6:7] -; GISEL-NEXT: v_add_u32_e32 v13, 26, v11 -; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v12 -; GISEL-NEXT: v_sub_u32_e32 v11, 64, v13 -; GISEL-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc -; GISEL-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc -; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v12 -; GISEL-NEXT: v_lshrrev_b64 v[4:5], v13, -1 +; GISEL-NEXT: s_cbranch_execz .LBB0_7 +; GISEL-NEXT: ; %bb.6: ; %itofp-sw-default +; GISEL-NEXT: v_sub_u32_e32 v4, 0x66, v5 +; GISEL-NEXT: v_sub_u32_e32 v11, 64, v4 +; GISEL-NEXT: v_lshrrev_b64 v[9:10], v4, v[0:1] +; GISEL-NEXT: v_lshlrev_b64 v[11:12], v11, v[2:3] +; GISEL-NEXT: v_subrev_u32_e32 v13, 64, v4 +; GISEL-NEXT: v_or_b32_e32 v11, v9, v11 +; GISEL-NEXT: v_or_b32_e32 v12, v10, v12 +; GISEL-NEXT: v_lshrrev_b64 v[9:10], v13, v[2:3] +; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v4 +; GISEL-NEXT: v_add_u32_e32 v5, 26, v5 +; GISEL-NEXT: v_cndmask_b32_e32 v9, v9, v11, vcc +; GISEL-NEXT: v_cndmask_b32_e32 v10, v10, v12, vcc +; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4 +; GISEL-NEXT: v_sub_u32_e32 v11, 64, v5 +; GISEL-NEXT: v_cndmask_b32_e32 v13, v9, v0, vcc +; GISEL-NEXT: v_cndmask_b32_e32 v4, v10, v1, vcc +; GISEL-NEXT: v_lshrrev_b64 v[9:10], v5, -1 ; GISEL-NEXT: v_lshlrev_b64 v[11:12], v11, -1 -; GISEL-NEXT: v_subrev_u32_e32 v14, 64, v13 -; GISEL-NEXT: v_or_b32_e32 v15, v4, v11 -; GISEL-NEXT: v_or_b32_e32 v16, v5, v12 +; GISEL-NEXT: v_subrev_u32_e32 v14, 64, v5 +; GISEL-NEXT: v_or_b32_e32 v15, v9, v11 +; GISEL-NEXT: v_or_b32_e32 v16, v10, v12 ; GISEL-NEXT: v_lshrrev_b64 v[11:12], v14, -1 -; GISEL-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc -; GISEL-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc -; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v13 +; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v5 ; GISEL-NEXT: v_cndmask_b32_e32 v11, v11, v15, vcc ; GISEL-NEXT: v_cndmask_b32_e32 v12, v12, v16, vcc -; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v13 -; GISEL-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc -; GISEL-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc -; GISEL-NEXT: v_cndmask_b32_e64 v11, v11, -1, s[4:5] -; GISEL-NEXT: v_cndmask_b32_e64 v12, v12, -1, s[4:5] -; GISEL-NEXT: v_and_b32_e32 v4, v4, v6 -; GISEL-NEXT: v_and_b32_e32 v5, v5, v7 -; GISEL-NEXT: v_and_or_b32 v4, v11, v0, v4 -; GISEL-NEXT: v_and_or_b32 v5, v12, v1, v5 -; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] -; GISEL-NEXT: s_andn2_b64 s[4:5], s[10:11], exec -; GISEL-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc -; GISEL-NEXT: s_and_b64 s[10:11], exec, 0 -; GISEL-NEXT: v_or_b32_e32 v2, v2, v4 -; GISEL-NEXT: s_or_b64 s[10:11], s[4:5], s[10:11] -; GISEL-NEXT: .LBB0_10: ; %Flow5 +; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v5 +; GISEL-NEXT: v_cndmask_b32_e32 v9, 0, v9, vcc +; GISEL-NEXT: v_cndmask_b32_e32 v10, 0, v10, vcc +; GISEL-NEXT: v_cndmask_b32_e64 v5, v11, -1, s[4:5] +; GISEL-NEXT: v_cndmask_b32_e64 v11, v12, -1, s[4:5] +; GISEL-NEXT: v_and_b32_e32 v2, v9, v2 +; GISEL-NEXT: v_and_b32_e32 v3, v10, v3 +; GISEL-NEXT: v_and_or_b32 v0, v5, v0, v2 +; GISEL-NEXT: v_and_or_b32 v1, v11, v1, v3 +; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] +; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc +; GISEL-NEXT: v_or_b32_e32 v3, v13, v0 +; GISEL-NEXT: v_mov_b32_e32 v0, v3 +; GISEL-NEXT: v_mov_b32_e32 v1, v4 +; GISEL-NEXT: v_mov_b32_e32 v2, v5 +; GISEL-NEXT: v_mov_b32_e32 v3, v6 +; GISEL-NEXT: .LBB0_7: ; %Flow1 ; GISEL-NEXT: s_or_b64 exec, exec, s[12:13] -; GISEL-NEXT: s_and_saveexec_b64 s[4:5], s[10:11] -; GISEL-NEXT: ; %bb.11: ; %itofp-sw-bb -; GISEL-NEXT: v_lshlrev_b64 v[2:3], 1, v[0:1] -; GISEL-NEXT: ; %bb.12: ; %itofp-sw-epilog +; GISEL-NEXT: .LBB0_8: ; %Flow2 +; GISEL-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11] +; GISEL-NEXT: ; %bb.9: ; %itofp-sw-bb +; GISEL-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1] +; GISEL-NEXT: ; %bb.10: ; %itofp-sw-epilog ; GISEL-NEXT: s_or_b64 exec, exec, s[4:5] -; GISEL-NEXT: v_bfe_u32 v0, v2, 2, 1 -; GISEL-NEXT: v_or_b32_e32 v0, v2, v0 +; GISEL-NEXT: v_bfe_u32 v2, v0, 2, 1 +; GISEL-NEXT: v_or_b32_e32 v0, v0, v2 ; GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 1, v0 -; GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc -; GISEL-NEXT: v_lshrrev_b64 v[2:3], 2, v[0:1] -; GISEL-NEXT: v_and_b32_e32 v3, 0x4000000, v0 -; GISEL-NEXT: v_mov_b32_e32 v4, 0 -; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[3:4] +; GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc +; GISEL-NEXT: v_and_b32_e32 v2, 0x4000000, v0 +; GISEL-NEXT: v_mov_b32_e32 v3, 0 +; GISEL-NEXT: v_lshrrev_b64 v[4:5], 2, v[0:1] +; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3] ; GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc -; GISEL-NEXT: ; %bb.13: ; %itofp-if-then20 -; GISEL-NEXT: v_lshrrev_b64 v[2:3], 3, v[0:1] -; GISEL-NEXT: v_mov_b32_e32 v9, v10 -; GISEL-NEXT: ; %bb.14: ; %Flow +; GISEL-NEXT: ; %bb.11: ; %itofp-if-then20 +; GISEL-NEXT: v_lshrrev_b64 v[4:5], 3, v[0:1] +; GISEL-NEXT: v_mov_b32_e32 v7, v8 +; GISEL-NEXT: ; %bb.12: ; %Flow ; GISEL-NEXT: s_or_b64 exec, exec, s[4:5] -; GISEL-NEXT: .LBB0_15: ; %Flow7 +; GISEL-NEXT: .LBB0_13: ; %Flow4 ; GISEL-NEXT: s_or_b64 exec, exec, s[8:9] -; GISEL-NEXT: v_and_b32_e32 v0, 0x80000000, v8 -; GISEL-NEXT: v_lshl_add_u32 v1, v9, 23, 1.0 -; GISEL-NEXT: v_and_b32_e32 v2, 0x7fffff, v2 +; GISEL-NEXT: v_and_b32_e32 v0, 0x80000000, v6 +; GISEL-NEXT: v_lshl_add_u32 v1, v7, 23, 1.0 +; GISEL-NEXT: v_and_b32_e32 v2, 0x7fffff, v4 ; GISEL-NEXT: v_or3_b32 v4, v2, v0, v1 -; GISEL-NEXT: .LBB0_16: ; %Flow8 +; GISEL-NEXT: .LBB0_14: ; %Flow5 ; GISEL-NEXT: s_or_b64 exec, exec, s[6:7] ; GISEL-NEXT: v_mov_b32_e32 v0, v4 ; GISEL-NEXT: s_setpc_b64 s[30:31] @@ -305,7 +275,7 @@ define float @uitofp_i128_to_f32(i128 %x) { ; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; SDAG-NEXT: v_mov_b32_e32 v4, 0 ; SDAG-NEXT: s_and_saveexec_b64 s[6:7], vcc -; SDAG-NEXT: s_cbranch_execz .LBB1_16 +; SDAG-NEXT: s_cbranch_execz .LBB1_14 ; SDAG-NEXT: ; %bb.1: ; %itofp-if-end ; SDAG-NEXT: v_ffbh_u32_e32 v4, v2 ; SDAG-NEXT: v_add_u32_e32 v4, 32, v4 @@ -317,113 +287,99 @@ define float @uitofp_i128_to_f32(i128 %x) { ; SDAG-NEXT: v_min_u32_e32 v5, v5, v6 ; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3] ; SDAG-NEXT: v_add_u32_e32 v5, 64, v5 -; SDAG-NEXT: v_cndmask_b32_e32 v8, v5, v4, vcc -; SDAG-NEXT: v_sub_u32_e32 v6, 0x7f, v8 -; SDAG-NEXT: v_mov_b32_e32 v5, v1 -; SDAG-NEXT: v_cmp_gt_i32_e32 vcc, 25, v6 -; SDAG-NEXT: v_mov_b32_e32 v4, v0 -; SDAG-NEXT: ; implicit-def: $vgpr9 +; SDAG-NEXT: v_cndmask_b32_e32 v6, v5, v4, vcc +; SDAG-NEXT: v_sub_u32_e32 v5, 0x80, v6 +; SDAG-NEXT: v_sub_u32_e32 v4, 0x7f, v6 +; SDAG-NEXT: v_cmp_gt_i32_e32 vcc, 25, v5 +; SDAG-NEXT: ; implicit-def: $vgpr7 ; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc ; SDAG-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; SDAG-NEXT: ; %bb.2: ; %itofp-if-else -; SDAG-NEXT: v_add_u32_e32 v2, 0xffffff98, v8 +; SDAG-NEXT: v_add_u32_e32 v2, 0xffffff98, v6 ; SDAG-NEXT: v_lshlrev_b64 v[0:1], v2, v[0:1] ; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v2 -; SDAG-NEXT: v_cndmask_b32_e32 v9, 0, v0, vcc +; SDAG-NEXT: v_cndmask_b32_e32 v7, 0, v0, vcc +; SDAG-NEXT: ; implicit-def: $vgpr5 ; SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 -; SDAG-NEXT: ; implicit-def: $vgpr8 +; SDAG-NEXT: ; implicit-def: $vgpr6 ; SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3 -; SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5 -; SDAG-NEXT: ; %bb.3: ; %Flow6 +; SDAG-NEXT: ; %bb.3: ; %Flow3 ; SDAG-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] -; SDAG-NEXT: s_cbranch_execz .LBB1_15 +; SDAG-NEXT: s_cbranch_execz .LBB1_13 ; SDAG-NEXT: ; %bb.4: ; %NodeBlock -; SDAG-NEXT: v_sub_u32_e32 v7, 0x80, v8 -; SDAG-NEXT: v_cmp_lt_i32_e32 vcc, 25, v7 -; SDAG-NEXT: s_mov_b64 s[10:11], 0 -; SDAG-NEXT: s_mov_b64 s[4:5], 0 +; SDAG-NEXT: v_cmp_lt_i32_e32 vcc, 25, v5 +; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc +; SDAG-NEXT: s_xor_b64 s[10:11], exec, s[4:5] +; SDAG-NEXT: s_cbranch_execz .LBB1_8 +; SDAG-NEXT: ; %bb.5: ; %LeafBlock +; SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 26, v5 ; SDAG-NEXT: s_and_saveexec_b64 s[12:13], vcc -; SDAG-NEXT: s_xor_b64 s[12:13], exec, s[12:13] -; SDAG-NEXT: ; %bb.5: ; %LeafBlock1 -; SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 26, v7 -; SDAG-NEXT: s_and_b64 s[4:5], vcc, exec -; SDAG-NEXT: ; %bb.6: ; %Flow3 -; SDAG-NEXT: s_andn2_saveexec_b64 s[12:13], s[12:13] -; SDAG-NEXT: ; %bb.7: ; %LeafBlock -; SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 25, v7 -; SDAG-NEXT: s_andn2_b64 s[4:5], s[4:5], exec -; SDAG-NEXT: s_and_b64 s[14:15], vcc, exec -; SDAG-NEXT: s_mov_b64 s[10:11], exec -; SDAG-NEXT: s_or_b64 s[4:5], s[4:5], s[14:15] -; SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5 -; SDAG-NEXT: ; %bb.8: ; %Flow4 -; SDAG-NEXT: s_or_b64 exec, exec, s[12:13] -; SDAG-NEXT: s_and_saveexec_b64 s[12:13], s[4:5] -; SDAG-NEXT: s_xor_b64 s[12:13], exec, s[12:13] -; SDAG-NEXT: s_cbranch_execz .LBB1_10 -; SDAG-NEXT: ; %bb.9: ; %itofp-sw-default -; SDAG-NEXT: v_sub_u32_e32 v11, 0x66, v8 +; SDAG-NEXT: s_cbranch_execz .LBB1_7 +; SDAG-NEXT: ; %bb.6: ; %itofp-sw-default +; SDAG-NEXT: v_sub_u32_e32 v11, 0x66, v6 ; SDAG-NEXT: v_sub_u32_e32 v9, 64, v11 -; SDAG-NEXT: v_lshrrev_b64 v[4:5], v11, v[0:1] +; SDAG-NEXT: v_lshrrev_b64 v[7:8], v11, v[0:1] ; SDAG-NEXT: v_lshlrev_b64 v[9:10], v9, v[2:3] -; SDAG-NEXT: v_sub_u32_e32 v12, 38, v8 -; SDAG-NEXT: v_or_b32_e32 v10, v5, v10 -; SDAG-NEXT: v_or_b32_e32 v9, v4, v9 -; SDAG-NEXT: v_lshrrev_b64 v[4:5], v12, v[2:3] +; SDAG-NEXT: v_sub_u32_e32 v12, 38, v6 +; SDAG-NEXT: v_or_b32_e32 v10, v8, v10 +; SDAG-NEXT: v_or_b32_e32 v9, v7, v9 +; SDAG-NEXT: v_lshrrev_b64 v[7:8], v12, v[2:3] ; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v11 -; SDAG-NEXT: v_add_u32_e32 v13, 26, v8 -; SDAG-NEXT: v_cndmask_b32_e32 v5, v5, v10, vcc +; SDAG-NEXT: v_add_u32_e32 v13, 26, v6 +; SDAG-NEXT: v_cndmask_b32_e32 v8, v8, v10, vcc ; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v11 -; SDAG-NEXT: v_cndmask_b32_e32 v4, v4, v9, vcc +; SDAG-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc ; SDAG-NEXT: v_lshrrev_b64 v[9:10], v12, v[0:1] ; SDAG-NEXT: v_lshlrev_b64 v[11:12], v13, v[2:3] -; SDAG-NEXT: v_subrev_u32_e32 v8, 38, v8 -; SDAG-NEXT: v_cndmask_b32_e64 v14, v4, v0, s[4:5] -; SDAG-NEXT: v_or_b32_e32 v4, v12, v10 -; SDAG-NEXT: v_or_b32_e32 v10, v11, v9 -; SDAG-NEXT: v_lshlrev_b64 v[8:9], v8, v[0:1] +; SDAG-NEXT: v_subrev_u32_e32 v6, 38, v6 +; SDAG-NEXT: v_cndmask_b32_e64 v14, v7, v0, s[4:5] +; SDAG-NEXT: v_lshlrev_b64 v[6:7], v6, v[0:1] +; SDAG-NEXT: v_cndmask_b32_e64 v8, v8, v1, s[4:5] +; SDAG-NEXT: v_or_b32_e32 v10, v12, v10 +; SDAG-NEXT: v_or_b32_e32 v9, v11, v9 ; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v13 -; SDAG-NEXT: v_cndmask_b32_e64 v5, v5, v1, s[4:5] -; SDAG-NEXT: v_cndmask_b32_e32 v4, v9, v4, vcc +; SDAG-NEXT: v_lshlrev_b64 v[0:1], v13, v[0:1] +; SDAG-NEXT: v_cndmask_b32_e32 v7, v7, v10, vcc ; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v13 -; SDAG-NEXT: v_cndmask_b32_e64 v9, v4, v3, s[4:5] -; SDAG-NEXT: v_lshlrev_b64 v[3:4], v13, v[0:1] -; SDAG-NEXT: v_cndmask_b32_e32 v8, v8, v10, vcc -; SDAG-NEXT: v_cndmask_b32_e64 v2, v8, v2, s[4:5] -; SDAG-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc -; SDAG-NEXT: v_cndmask_b32_e32 v8, 0, v3, vcc -; SDAG-NEXT: v_or_b32_e32 v3, v4, v9 -; SDAG-NEXT: v_or_b32_e32 v2, v8, v2 -; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3] -; SDAG-NEXT: s_andn2_b64 s[10:11], s[10:11], exec -; SDAG-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc -; SDAG-NEXT: v_or_b32_e32 v4, v14, v2 -; SDAG-NEXT: .LBB1_10: ; %Flow5 +; SDAG-NEXT: v_cndmask_b32_e32 v6, v6, v9, vcc +; SDAG-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[4:5] +; SDAG-NEXT: v_cndmask_b32_e64 v2, v6, v2, s[4:5] +; SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc +; SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc +; SDAG-NEXT: v_or_b32_e32 v1, v1, v3 +; SDAG-NEXT: v_or_b32_e32 v0, v0, v2 +; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] +; SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc +; SDAG-NEXT: v_or_b32_e32 v7, v14, v0 +; SDAG-NEXT: v_mov_b32_e32 v0, v7 +; SDAG-NEXT: v_mov_b32_e32 v1, v8 +; SDAG-NEXT: .LBB1_7: ; %Flow1 ; SDAG-NEXT: s_or_b64 exec, exec, s[12:13] -; SDAG-NEXT: s_and_saveexec_b64 s[4:5], s[10:11] -; SDAG-NEXT: ; %bb.11: ; %itofp-sw-bb -; SDAG-NEXT: v_lshlrev_b64 v[4:5], 1, v[0:1] -; SDAG-NEXT: ; %bb.12: ; %itofp-sw-epilog +; SDAG-NEXT: .LBB1_8: ; %Flow2 +; SDAG-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11] +; SDAG-NEXT: ; %bb.9: ; %itofp-sw-bb +; SDAG-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1] +; SDAG-NEXT: ; %bb.10: ; %itofp-sw-epilog ; SDAG-NEXT: s_or_b64 exec, exec, s[4:5] -; SDAG-NEXT: v_lshrrev_b32_e32 v0, 2, v4 -; SDAG-NEXT: v_and_or_b32 v0, v0, 1, v4 +; SDAG-NEXT: v_lshrrev_b32_e32 v2, 2, v0 +; SDAG-NEXT: v_and_or_b32 v0, v2, 1, v0 ; SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 1, v0 -; SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v5, vcc +; SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc ; SDAG-NEXT: v_and_b32_e32 v2, 0x4000000, v0 ; SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2 -; SDAG-NEXT: v_alignbit_b32 v9, v1, v0, 2 +; SDAG-NEXT: v_alignbit_b32 v7, v1, v0, 2 ; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc -; SDAG-NEXT: ; %bb.13: ; %itofp-if-then20 -; SDAG-NEXT: v_alignbit_b32 v9, v1, v0, 3 -; SDAG-NEXT: v_mov_b32_e32 v6, v7 -; SDAG-NEXT: ; %bb.14: ; %Flow +; SDAG-NEXT: ; %bb.11: ; %itofp-if-then20 +; SDAG-NEXT: v_alignbit_b32 v7, v1, v0, 3 +; SDAG-NEXT: v_mov_b32_e32 v4, v5 +; SDAG-NEXT: ; %bb.12: ; %Flow ; SDAG-NEXT: s_or_b64 exec, exec, s[4:5] -; SDAG-NEXT: .LBB1_15: ; %Flow7 +; SDAG-NEXT: .LBB1_13: ; %Flow4 ; SDAG-NEXT: s_or_b64 exec, exec, s[8:9] -; SDAG-NEXT: v_and_b32_e32 v0, 0x7fffff, v9 -; SDAG-NEXT: v_lshl_or_b32 v0, v6, 23, v0 +; SDAG-NEXT: v_and_b32_e32 v0, 0x7fffff, v7 +; SDAG-NEXT: v_lshl_or_b32 v0, v4, 23, v0 ; SDAG-NEXT: v_add_u32_e32 v4, 1.0, v0 -; SDAG-NEXT: .LBB1_16: ; %Flow8 +; SDAG-NEXT: .LBB1_14: ; %Flow5 ; SDAG-NEXT: s_or_b64 exec, exec, s[6:7] ; SDAG-NEXT: v_mov_b32_e32 v0, v4 ; SDAG-NEXT: s_setpc_b64 s[30:31] @@ -431,144 +387,124 @@ define float @uitofp_i128_to_f32(i128 %x) { ; GISEL-LABEL: uitofp_i128_to_f32: ; GISEL: ; %bb.0: ; %itofp-entry ; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GISEL-NEXT: v_mov_b32_e32 v4, v2 -; GISEL-NEXT: v_mov_b32_e32 v5, v3 -; GISEL-NEXT: v_or_b32_e32 v2, v0, v4 -; GISEL-NEXT: v_or_b32_e32 v3, v1, v5 -; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3] +; GISEL-NEXT: v_or_b32_e32 v4, v0, v2 +; GISEL-NEXT: v_or_b32_e32 v5, v1, v3 +; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GISEL-NEXT: s_mov_b32 s4, 0 -; GISEL-NEXT: v_mov_b32_e32 v2, s4 +; GISEL-NEXT: v_mov_b32_e32 v4, s4 ; GISEL-NEXT: s_and_saveexec_b64 s[6:7], vcc -; GISEL-NEXT: s_cbranch_execz .LBB1_16 +; GISEL-NEXT: s_cbranch_execz .LBB1_14 ; GISEL-NEXT: ; %bb.1: ; %itofp-if-end -; GISEL-NEXT: v_ffbh_u32_e32 v3, v0 -; GISEL-NEXT: v_ffbh_u32_e32 v2, v1 -; GISEL-NEXT: v_add_u32_e32 v3, 32, v3 -; GISEL-NEXT: v_ffbh_u32_e32 v6, v4 -; GISEL-NEXT: v_min_u32_e32 v2, v2, v3 -; GISEL-NEXT: v_ffbh_u32_e32 v3, v5 +; GISEL-NEXT: v_ffbh_u32_e32 v5, v0 +; GISEL-NEXT: v_ffbh_u32_e32 v4, v1 +; GISEL-NEXT: v_add_u32_e32 v5, 32, v5 +; GISEL-NEXT: v_ffbh_u32_e32 v6, v2 +; GISEL-NEXT: v_min_u32_e32 v4, v4, v5 +; GISEL-NEXT: v_ffbh_u32_e32 v5, v3 ; GISEL-NEXT: v_add_u32_e32 v6, 32, v6 -; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[4:5] -; GISEL-NEXT: v_add_u32_e32 v2, 64, v2 -; GISEL-NEXT: v_min_u32_e32 v3, v3, v6 -; GISEL-NEXT: v_cndmask_b32_e32 v12, v3, v2, vcc -; GISEL-NEXT: v_sub_u32_e32 v10, 0x7f, v12 -; GISEL-NEXT: v_cmp_ge_i32_e32 vcc, 24, v10 -; GISEL-NEXT: ; implicit-def: $vgpr2 +; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3] +; GISEL-NEXT: v_add_u32_e32 v4, 64, v4 +; GISEL-NEXT: v_min_u32_e32 v5, v5, v6 +; GISEL-NEXT: v_cndmask_b32_e32 v5, v5, v4, vcc +; GISEL-NEXT: v_sub_u32_e32 v7, 0x80, v5 +; GISEL-NEXT: v_sub_u32_e32 v6, 0x7f, v5 +; GISEL-NEXT: v_cmp_ge_i32_e32 vcc, 24, v7 +; GISEL-NEXT: ; implicit-def: $vgpr4 ; GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GISEL-NEXT: ; %bb.2: ; %itofp-if-else -; GISEL-NEXT: v_add_u32_e32 v2, 0xffffff98, v12 +; GISEL-NEXT: v_add_u32_e32 v2, 0xffffff98, v5 ; GISEL-NEXT: v_lshlrev_b64 v[0:1], v2, v[0:1] ; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v2 -; GISEL-NEXT: v_cndmask_b32_e32 v2, 0, v0, vcc +; GISEL-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc +; GISEL-NEXT: ; implicit-def: $vgpr7 ; GISEL-NEXT: ; implicit-def: $vgpr0 -; GISEL-NEXT: ; implicit-def: $vgpr12 -; GISEL-NEXT: ; implicit-def: $vgpr4 -; GISEL-NEXT: ; %bb.3: ; %Flow6 +; GISEL-NEXT: ; implicit-def: $vgpr5 +; GISEL-NEXT: ; implicit-def: $vgpr2 +; GISEL-NEXT: ; %bb.3: ; %Flow3 ; GISEL-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] -; GISEL-NEXT: s_cbranch_execz .LBB1_15 +; GISEL-NEXT: s_cbranch_execz .LBB1_13 ; GISEL-NEXT: ; %bb.4: ; %NodeBlock -; GISEL-NEXT: v_sub_u32_e32 v11, 0x80, v12 -; GISEL-NEXT: v_cmp_le_i32_e32 vcc, 26, v11 -; GISEL-NEXT: s_mov_b64 s[10:11], 0 -; GISEL-NEXT: s_mov_b64 s[4:5], 0 +; GISEL-NEXT: v_cmp_le_i32_e32 vcc, 26, v7 +; GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GISEL-NEXT: s_xor_b64 s[10:11], exec, s[4:5] +; GISEL-NEXT: s_cbranch_execz .LBB1_8 +; GISEL-NEXT: ; %bb.5: ; %LeafBlock +; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 26, v7 ; GISEL-NEXT: s_and_saveexec_b64 s[12:13], vcc -; GISEL-NEXT: s_xor_b64 s[12:13], exec, s[12:13] -; GISEL-NEXT: ; %bb.5: ; %LeafBlock1 -; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 26, v11 -; GISEL-NEXT: s_andn2_b64 s[4:5], 0, exec -; GISEL-NEXT: s_and_b64 s[14:15], exec, vcc -; GISEL-NEXT: s_or_b64 s[4:5], s[4:5], s[14:15] -; GISEL-NEXT: ; %bb.6: ; %Flow3 -; GISEL-NEXT: s_andn2_saveexec_b64 s[12:13], s[12:13] -; GISEL-NEXT: ; %bb.7: ; %LeafBlock -; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 25, v11 -; GISEL-NEXT: s_andn2_b64 s[10:11], 0, exec -; GISEL-NEXT: s_and_b64 s[14:15], exec, -1 -; GISEL-NEXT: s_or_b64 s[10:11], s[10:11], s[14:15] -; GISEL-NEXT: s_andn2_b64 s[4:5], s[4:5], exec -; GISEL-NEXT: s_and_b64 s[14:15], exec, vcc -; GISEL-NEXT: s_or_b64 s[4:5], s[4:5], s[14:15] -; GISEL-NEXT: ; %bb.8: ; %Flow4 -; GISEL-NEXT: s_or_b64 exec, exec, s[12:13] -; GISEL-NEXT: v_mov_b32_e32 v9, v3 -; GISEL-NEXT: v_mov_b32_e32 v7, v1 -; GISEL-NEXT: v_mov_b32_e32 v6, v0 -; GISEL-NEXT: v_mov_b32_e32 v8, v2 -; GISEL-NEXT: s_and_saveexec_b64 s[12:13], s[4:5] -; GISEL-NEXT: s_xor_b64 s[12:13], exec, s[12:13] -; GISEL-NEXT: s_cbranch_execz .LBB1_10 -; GISEL-NEXT: ; %bb.9: ; %itofp-sw-default -; GISEL-NEXT: v_sub_u32_e32 v8, 0x66, v12 -; GISEL-NEXT: v_sub_u32_e32 v6, 64, v8 -; GISEL-NEXT: v_lshrrev_b64 v[2:3], v8, v[0:1] -; GISEL-NEXT: v_lshlrev_b64 v[6:7], v6, v[4:5] -; GISEL-NEXT: v_subrev_u32_e32 v9, 64, v8 -; GISEL-NEXT: v_or_b32_e32 v6, v2, v6 -; GISEL-NEXT: v_or_b32_e32 v7, v3, v7 -; GISEL-NEXT: v_lshrrev_b64 v[2:3], v9, v[4:5] -; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v8 -; GISEL-NEXT: v_add_u32_e32 v12, 26, v12 -; GISEL-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc -; GISEL-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc -; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v8 -; GISEL-NEXT: v_sub_u32_e32 v8, 64, v12 -; GISEL-NEXT: v_cndmask_b32_e32 v6, v2, v0, vcc -; GISEL-NEXT: v_cndmask_b32_e32 v7, v3, v1, vcc -; GISEL-NEXT: v_lshrrev_b64 v[2:3], v12, -1 -; GISEL-NEXT: v_lshlrev_b64 v[8:9], v8, -1 -; GISEL-NEXT: v_subrev_u32_e32 v13, 64, v12 -; GISEL-NEXT: v_or_b32_e32 v14, v2, v8 -; GISEL-NEXT: v_or_b32_e32 v15, v3, v9 -; GISEL-NEXT: v_lshrrev_b64 v[8:9], v13, -1 -; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v12 -; GISEL-NEXT: v_cndmask_b32_e32 v8, v8, v14, vcc -; GISEL-NEXT: v_cndmask_b32_e32 v9, v9, v15, vcc -; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v12 -; GISEL-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc -; GISEL-NEXT: v_cndmask_b32_e32 v3, 0, v3, vcc -; GISEL-NEXT: v_cndmask_b32_e64 v8, v8, -1, s[4:5] -; GISEL-NEXT: v_cndmask_b32_e64 v9, v9, -1, s[4:5] -; GISEL-NEXT: v_and_b32_e32 v2, v2, v4 -; GISEL-NEXT: v_and_b32_e32 v3, v3, v5 -; GISEL-NEXT: v_and_or_b32 v2, v8, v0, v2 -; GISEL-NEXT: v_and_or_b32 v3, v9, v1, v3 -; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3] -; GISEL-NEXT: s_andn2_b64 s[4:5], s[10:11], exec -; GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc -; GISEL-NEXT: s_and_b64 s[10:11], exec, 0 -; GISEL-NEXT: v_or_b32_e32 v6, v6, v2 -; GISEL-NEXT: s_or_b64 s[10:11], s[4:5], s[10:11] -; GISEL-NEXT: .LBB1_10: ; %Flow5 +; GISEL-NEXT: s_cbranch_execz .LBB1_7 +; GISEL-NEXT: ; %bb.6: ; %itofp-sw-default +; GISEL-NEXT: v_sub_u32_e32 v4, 0x66, v5 +; GISEL-NEXT: v_sub_u32_e32 v10, 64, v4 +; GISEL-NEXT: v_lshrrev_b64 v[8:9], v4, v[0:1] +; GISEL-NEXT: v_lshlrev_b64 v[10:11], v10, v[2:3] +; GISEL-NEXT: v_subrev_u32_e32 v12, 64, v4 +; GISEL-NEXT: v_or_b32_e32 v10, v8, v10 +; GISEL-NEXT: v_or_b32_e32 v11, v9, v11 +; GISEL-NEXT: v_lshrrev_b64 v[8:9], v12, v[2:3] +; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v4 +; GISEL-NEXT: v_add_u32_e32 v5, 26, v5 +; GISEL-NEXT: v_cndmask_b32_e32 v8, v8, v10, vcc +; GISEL-NEXT: v_cndmask_b32_e32 v9, v9, v11, vcc +; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4 +; GISEL-NEXT: v_sub_u32_e32 v10, 64, v5 +; GISEL-NEXT: v_cndmask_b32_e32 v12, v8, v0, vcc +; GISEL-NEXT: v_cndmask_b32_e32 v4, v9, v1, vcc +; GISEL-NEXT: v_lshrrev_b64 v[8:9], v5, -1 +; GISEL-NEXT: v_lshlrev_b64 v[10:11], v10, -1 +; GISEL-NEXT: v_subrev_u32_e32 v13, 64, v5 +; GISEL-NEXT: v_or_b32_e32 v14, v8, v10 +; GISEL-NEXT: v_or_b32_e32 v15, v9, v11 +; GISEL-NEXT: v_lshrrev_b64 v[10:11], v13, -1 +; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v5 +; GISEL-NEXT: v_cndmask_b32_e32 v10, v10, v14, vcc +; GISEL-NEXT: v_cndmask_b32_e32 v11, v11, v15, vcc +; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v5 +; GISEL-NEXT: v_cndmask_b32_e32 v8, 0, v8, vcc +; GISEL-NEXT: v_cndmask_b32_e32 v9, 0, v9, vcc +; GISEL-NEXT: v_cndmask_b32_e64 v5, v10, -1, s[4:5] +; GISEL-NEXT: v_cndmask_b32_e64 v10, v11, -1, s[4:5] +; GISEL-NEXT: v_and_b32_e32 v2, v8, v2 +; GISEL-NEXT: v_and_b32_e32 v3, v9, v3 +; GISEL-NEXT: v_and_or_b32 v0, v5, v0, v2 +; GISEL-NEXT: v_and_or_b32 v1, v10, v1, v3 +; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] +; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc +; GISEL-NEXT: v_or_b32_e32 v3, v12, v0 +; GISEL-NEXT: v_mov_b32_e32 v0, v3 +; GISEL-NEXT: v_mov_b32_e32 v1, v4 +; GISEL-NEXT: v_mov_b32_e32 v2, v5 +; GISEL-NEXT: v_mov_b32_e32 v3, v6 +; GISEL-NEXT: .LBB1_7: ; %Flow1 ; GISEL-NEXT: s_or_b64 exec, exec, s[12:13] -; GISEL-NEXT: s_and_saveexec_b64 s[4:5], s[10:11] -; GISEL-NEXT: ; %bb.11: ; %itofp-sw-bb -; GISEL-NEXT: v_lshlrev_b64 v[6:7], 1, v[0:1] -; GISEL-NEXT: ; %bb.12: ; %itofp-sw-epilog +; GISEL-NEXT: .LBB1_8: ; %Flow2 +; GISEL-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11] +; GISEL-NEXT: ; %bb.9: ; %itofp-sw-bb +; GISEL-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1] +; GISEL-NEXT: ; %bb.10: ; %itofp-sw-epilog ; GISEL-NEXT: s_or_b64 exec, exec, s[4:5] -; GISEL-NEXT: v_bfe_u32 v0, v6, 2, 1 -; GISEL-NEXT: v_or_b32_e32 v0, v6, v0 +; GISEL-NEXT: v_bfe_u32 v2, v0, 2, 1 +; GISEL-NEXT: v_or_b32_e32 v0, v0, v2 ; GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 1, v0 -; GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v7, vcc -; GISEL-NEXT: v_lshrrev_b64 v[2:3], 2, v[0:1] -; GISEL-NEXT: v_and_b32_e32 v3, 0x4000000, v0 -; GISEL-NEXT: v_mov_b32_e32 v4, 0 -; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[3:4] +; GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc +; GISEL-NEXT: v_and_b32_e32 v2, 0x4000000, v0 +; GISEL-NEXT: v_mov_b32_e32 v3, 0 +; GISEL-NEXT: v_lshrrev_b64 v[4:5], 2, v[0:1] +; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3] ; GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc -; GISEL-NEXT: ; %bb.13: ; %itofp-if-then20 -; GISEL-NEXT: v_lshrrev_b64 v[2:3], 3, v[0:1] -; GISEL-NEXT: v_mov_b32_e32 v10, v11 -; GISEL-NEXT: ; %bb.14: ; %Flow +; GISEL-NEXT: ; %bb.11: ; %itofp-if-then20 +; GISEL-NEXT: v_lshrrev_b64 v[4:5], 3, v[0:1] +; GISEL-NEXT: v_mov_b32_e32 v6, v7 +; GISEL-NEXT: ; %bb.12: ; %Flow ; GISEL-NEXT: s_or_b64 exec, exec, s[4:5] -; GISEL-NEXT: .LBB1_15: ; %Flow7 +; GISEL-NEXT: .LBB1_13: ; %Flow4 ; GISEL-NEXT: s_or_b64 exec, exec, s[8:9] -; GISEL-NEXT: v_lshl_add_u32 v0, v10, 23, 1.0 +; GISEL-NEXT: v_lshl_add_u32 v0, v6, 23, 1.0 ; GISEL-NEXT: v_mov_b32_e32 v1, 0x7fffff -; GISEL-NEXT: v_and_or_b32 v2, v2, v1, v0 -; GISEL-NEXT: .LBB1_16: ; %Flow8 +; GISEL-NEXT: v_and_or_b32 v4, v4, v1, v0 +; GISEL-NEXT: .LBB1_14: ; %Flow5 ; GISEL-NEXT: s_or_b64 exec, exec, s[6:7] -; GISEL-NEXT: v_mov_b32_e32 v0, v2 +; GISEL-NEXT: v_mov_b32_e32 v0, v4 ; GISEL-NEXT: s_setpc_b64 s[30:31] %cvt = uitofp i128 %x to float ret float %cvt @@ -586,7 +522,7 @@ define double @sitofp_i128_to_f64(i128 %x) { ; SDAG-NEXT: v_mov_b32_e32 v0, 0 ; SDAG-NEXT: v_mov_b32_e32 v1, 0 ; SDAG-NEXT: s_and_saveexec_b64 s[6:7], vcc -; SDAG-NEXT: s_cbranch_execz .LBB2_16 +; SDAG-NEXT: s_cbranch_execz .LBB2_14 ; SDAG-NEXT: ; %bb.1: ; %itofp-if-end ; SDAG-NEXT: v_ashrrev_i32_e32 v0, 31, v3 ; SDAG-NEXT: v_xor_b32_e32 v4, v0, v4 @@ -607,128 +543,116 @@ define double @sitofp_i128_to_f64(i128 %x) { ; SDAG-NEXT: v_min_u32_e32 v1, v1, v2 ; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7] ; SDAG-NEXT: v_add_u32_e32 v1, 64, v1 -; SDAG-NEXT: v_cndmask_b32_e32 v11, v1, v0, vcc -; SDAG-NEXT: v_sub_u32_e32 v2, 0x7f, v11 -; SDAG-NEXT: v_cmp_gt_i32_e32 vcc, 54, v2 -; SDAG-NEXT: ; implicit-def: $vgpr8 +; SDAG-NEXT: v_cndmask_b32_e32 v9, v1, v0, vcc +; SDAG-NEXT: v_sub_u32_e32 v8, 0x80, v9 +; SDAG-NEXT: v_sub_u32_e32 v2, 0x7f, v9 +; SDAG-NEXT: v_cmp_gt_i32_e32 vcc, 54, v8 +; SDAG-NEXT: ; implicit-def: $vgpr10 ; SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 ; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc ; SDAG-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; SDAG-NEXT: ; %bb.2: ; %itofp-if-else -; SDAG-NEXT: v_add_u32_e32 v6, 0xffffffb5, v11 +; SDAG-NEXT: v_add_u32_e32 v6, 0xffffffb5, v9 ; SDAG-NEXT: v_lshlrev_b64 v[0:1], v6, v[4:5] ; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v6 -; SDAG-NEXT: v_cndmask_b32_e32 v8, 0, v1, vcc +; SDAG-NEXT: v_cndmask_b32_e32 v10, 0, v1, vcc ; SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc +; SDAG-NEXT: ; implicit-def: $vgpr8 ; SDAG-NEXT: ; implicit-def: $vgpr6_vgpr7 ; SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5 -; SDAG-NEXT: ; implicit-def: $vgpr11 -; SDAG-NEXT: ; %bb.3: ; %Flow6 +; SDAG-NEXT: ; implicit-def: $vgpr9 +; SDAG-NEXT: ; %bb.3: ; %Flow3 ; SDAG-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] -; SDAG-NEXT: s_cbranch_execz .LBB2_15 +; SDAG-NEXT: s_cbranch_execz .LBB2_13 ; SDAG-NEXT: ; %bb.4: ; %NodeBlock -; SDAG-NEXT: v_sub_u32_e32 v10, 0x80, v11 -; SDAG-NEXT: v_cmp_lt_i32_e32 vcc, 54, v10 -; SDAG-NEXT: s_mov_b64 s[10:11], 0 -; SDAG-NEXT: s_mov_b64 s[4:5], 0 +; SDAG-NEXT: v_cmp_lt_i32_e32 vcc, 54, v8 +; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc +; SDAG-NEXT: s_xor_b64 s[10:11], exec, s[4:5] +; SDAG-NEXT: s_cbranch_execz .LBB2_8 +; SDAG-NEXT: ; %bb.5: ; %LeafBlock +; SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 55, v8 ; SDAG-NEXT: s_and_saveexec_b64 s[12:13], vcc -; SDAG-NEXT: s_xor_b64 s[12:13], exec, s[12:13] -; SDAG-NEXT: ; %bb.5: ; %LeafBlock1 -; SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 55, v10 -; SDAG-NEXT: s_and_b64 s[4:5], vcc, exec -; SDAG-NEXT: ; %bb.6: ; %Flow3 -; SDAG-NEXT: s_andn2_saveexec_b64 s[12:13], s[12:13] -; SDAG-NEXT: ; %bb.7: ; %LeafBlock -; SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 54, v10 -; SDAG-NEXT: s_andn2_b64 s[4:5], s[4:5], exec -; SDAG-NEXT: s_and_b64 s[14:15], vcc, exec -; SDAG-NEXT: s_mov_b64 s[10:11], exec -; SDAG-NEXT: s_or_b64 s[4:5], s[4:5], s[14:15] -; SDAG-NEXT: ; %bb.8: ; %Flow4 -; SDAG-NEXT: s_or_b64 exec, exec, s[12:13] -; SDAG-NEXT: v_mov_b32_e32 v0, v4 -; SDAG-NEXT: v_mov_b32_e32 v9, v7 -; SDAG-NEXT: v_mov_b32_e32 v1, v5 -; SDAG-NEXT: v_mov_b32_e32 v8, v6 -; SDAG-NEXT: s_and_saveexec_b64 s[12:13], s[4:5] -; SDAG-NEXT: s_xor_b64 s[12:13], exec, s[12:13] -; SDAG-NEXT: s_cbranch_execz .LBB2_10 -; SDAG-NEXT: ; %bb.9: ; %itofp-sw-default -; SDAG-NEXT: v_sub_u32_e32 v12, 0x49, v11 -; SDAG-NEXT: v_sub_u32_e32 v8, 64, v12 +; SDAG-NEXT: s_cbranch_execz .LBB2_7 +; SDAG-NEXT: ; %bb.6: ; %itofp-sw-default +; SDAG-NEXT: v_sub_u32_e32 v12, 0x49, v9 +; SDAG-NEXT: v_sub_u32_e32 v10, 64, v12 ; SDAG-NEXT: v_lshrrev_b64 v[0:1], v12, v[4:5] -; SDAG-NEXT: v_lshlrev_b64 v[8:9], v8, v[6:7] -; SDAG-NEXT: v_sub_u32_e32 v13, 9, v11 -; SDAG-NEXT: v_or_b32_e32 v9, v1, v9 -; SDAG-NEXT: v_or_b32_e32 v8, v0, v8 +; SDAG-NEXT: v_lshlrev_b64 v[10:11], v10, v[6:7] +; SDAG-NEXT: v_sub_u32_e32 v13, 9, v9 +; SDAG-NEXT: v_or_b32_e32 v11, v1, v11 +; SDAG-NEXT: v_or_b32_e32 v10, v0, v10 ; SDAG-NEXT: v_lshrrev_b64 v[0:1], v13, v[6:7] ; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v12 -; SDAG-NEXT: v_cndmask_b32_e32 v1, v1, v9, vcc -; SDAG-NEXT: v_cndmask_b32_e32 v0, v0, v8, vcc -; SDAG-NEXT: v_lshrrev_b64 v[8:9], v12, v[6:7] +; SDAG-NEXT: v_add_u32_e32 v16, 55, v9 +; SDAG-NEXT: v_cndmask_b32_e32 v1, v1, v11, vcc ; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v12 -; SDAG-NEXT: v_cndmask_b32_e32 v9, 0, v9, vcc -; SDAG-NEXT: v_add_u32_e32 v9, 55, v11 +; SDAG-NEXT: v_cndmask_b32_e32 v0, v0, v10, vcc +; SDAG-NEXT: v_lshrrev_b64 v[10:11], v12, v[6:7] ; SDAG-NEXT: v_lshrrev_b64 v[12:13], v13, v[4:5] -; SDAG-NEXT: v_lshlrev_b64 v[14:15], v9, v[6:7] -; SDAG-NEXT: v_add_u32_e32 v11, -9, v11 +; SDAG-NEXT: v_lshlrev_b64 v[14:15], v16, v[6:7] +; SDAG-NEXT: v_add_u32_e32 v9, -9, v9 +; SDAG-NEXT: v_or_b32_e32 v15, v15, v13 ; SDAG-NEXT: v_or_b32_e32 v14, v14, v12 -; SDAG-NEXT: v_lshlrev_b64 v[11:12], v11, v[4:5] -; SDAG-NEXT: v_cndmask_b32_e32 v8, 0, v8, vcc -; SDAG-NEXT: v_or_b32_e32 v13, v15, v13 -; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v9 +; SDAG-NEXT: v_lshlrev_b64 v[12:13], v9, v[4:5] +; SDAG-NEXT: v_cndmask_b32_e32 v11, 0, v11, vcc +; SDAG-NEXT: v_cndmask_b32_e32 v10, 0, v10, vcc +; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v16 ; SDAG-NEXT: v_cndmask_b32_e64 v1, v1, v5, s[4:5] ; SDAG-NEXT: v_cndmask_b32_e64 v0, v0, v4, s[4:5] -; SDAG-NEXT: v_cndmask_b32_e32 v12, v12, v13, vcc -; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v9 -; SDAG-NEXT: v_cndmask_b32_e64 v13, v12, v7, s[4:5] -; SDAG-NEXT: v_cndmask_b32_e32 v14, v11, v14, vcc -; SDAG-NEXT: v_lshlrev_b64 v[11:12], v9, v[4:5] -; SDAG-NEXT: v_cndmask_b32_e64 v9, v14, v6, s[4:5] -; SDAG-NEXT: v_cndmask_b32_e32 v12, 0, v12, vcc -; SDAG-NEXT: v_cndmask_b32_e32 v11, 0, v11, vcc -; SDAG-NEXT: v_or_b32_e32 v12, v12, v13 -; SDAG-NEXT: v_or_b32_e32 v11, v11, v9 -; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[11:12] -; SDAG-NEXT: s_andn2_b64 s[10:11], s[10:11], exec -; SDAG-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc -; SDAG-NEXT: v_or_b32_e32 v0, v0, v9 -; SDAG-NEXT: .LBB2_10: ; %Flow5 +; SDAG-NEXT: v_cndmask_b32_e32 v9, v13, v15, vcc +; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v16 +; SDAG-NEXT: v_lshlrev_b64 v[4:5], v16, v[4:5] +; SDAG-NEXT: v_cndmask_b32_e64 v7, v9, v7, s[4:5] +; SDAG-NEXT: v_cndmask_b32_e32 v9, v12, v14, vcc +; SDAG-NEXT: v_cndmask_b32_e64 v6, v9, v6, s[4:5] +; SDAG-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc +; SDAG-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc +; SDAG-NEXT: v_or_b32_e32 v5, v5, v7 +; SDAG-NEXT: v_or_b32_e32 v4, v4, v6 +; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] +; SDAG-NEXT: v_mov_b32_e32 v6, v10 +; SDAG-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc +; SDAG-NEXT: v_or_b32_e32 v0, v0, v4 +; SDAG-NEXT: v_mov_b32_e32 v5, v1 +; SDAG-NEXT: v_mov_b32_e32 v4, v0 +; SDAG-NEXT: v_mov_b32_e32 v7, v11 +; SDAG-NEXT: .LBB2_7: ; %Flow1 ; SDAG-NEXT: s_or_b64 exec, exec, s[12:13] -; SDAG-NEXT: s_and_saveexec_b64 s[4:5], s[10:11] -; SDAG-NEXT: ; %bb.11: ; %itofp-sw-bb -; SDAG-NEXT: v_lshlrev_b64 v[8:9], 1, v[6:7] -; SDAG-NEXT: v_lshrrev_b32_e32 v6, 31, v5 -; SDAG-NEXT: v_lshlrev_b64 v[0:1], 1, v[4:5] -; SDAG-NEXT: v_or_b32_e32 v8, v8, v6 -; SDAG-NEXT: ; %bb.12: ; %itofp-sw-epilog +; SDAG-NEXT: .LBB2_8: ; %Flow2 +; SDAG-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11] +; SDAG-NEXT: ; %bb.9: ; %itofp-sw-bb +; SDAG-NEXT: v_lshlrev_b64 v[6:7], 1, v[6:7] +; SDAG-NEXT: v_lshrrev_b32_e32 v0, 31, v5 +; SDAG-NEXT: v_lshlrev_b64 v[4:5], 1, v[4:5] +; SDAG-NEXT: v_or_b32_e32 v6, v6, v0 +; SDAG-NEXT: ; %bb.10: ; %itofp-sw-epilog ; SDAG-NEXT: s_or_b64 exec, exec, s[4:5] -; SDAG-NEXT: v_lshrrev_b32_e32 v4, 2, v0 -; SDAG-NEXT: v_and_or_b32 v0, v4, 1, v0 +; SDAG-NEXT: v_lshrrev_b32_e32 v0, 2, v4 +; SDAG-NEXT: v_and_or_b32 v0, v0, 1, v4 ; SDAG-NEXT: v_add_co_u32_e32 v4, vcc, 1, v0 -; SDAG-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc -; SDAG-NEXT: v_addc_co_u32_e32 v6, vcc, 0, v8, vcc +; SDAG-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v5, vcc +; SDAG-NEXT: v_addc_co_u32_e32 v6, vcc, 0, v6, vcc ; SDAG-NEXT: v_lshrrev_b64 v[0:1], 2, v[4:5] ; SDAG-NEXT: v_lshlrev_b32_e32 v7, 30, v6 -; SDAG-NEXT: v_or_b32_e32 v8, v1, v7 +; SDAG-NEXT: v_or_b32_e32 v10, v1, v7 ; SDAG-NEXT: v_and_b32_e32 v1, 0x800000, v5 ; SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1 ; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc -; SDAG-NEXT: ; %bb.13: ; %itofp-if-then20 +; SDAG-NEXT: ; %bb.11: ; %itofp-if-then20 ; SDAG-NEXT: v_lshrrev_b64 v[0:1], 3, v[4:5] ; SDAG-NEXT: v_lshlrev_b32_e32 v2, 29, v6 -; SDAG-NEXT: v_or_b32_e32 v8, v1, v2 -; SDAG-NEXT: v_mov_b32_e32 v2, v10 -; SDAG-NEXT: ; %bb.14: ; %Flow +; SDAG-NEXT: v_or_b32_e32 v10, v1, v2 +; SDAG-NEXT: v_mov_b32_e32 v2, v8 +; SDAG-NEXT: ; %bb.12: ; %Flow ; SDAG-NEXT: s_or_b64 exec, exec, s[4:5] -; SDAG-NEXT: .LBB2_15: ; %Flow7 +; SDAG-NEXT: .LBB2_13: ; %Flow4 ; SDAG-NEXT: s_or_b64 exec, exec, s[8:9] ; SDAG-NEXT: v_and_b32_e32 v1, 0x80000000, v3 ; SDAG-NEXT: v_mov_b32_e32 v3, 0x3ff00000 ; SDAG-NEXT: v_lshl_add_u32 v2, v2, 20, v3 -; SDAG-NEXT: v_and_b32_e32 v3, 0xfffff, v8 +; SDAG-NEXT: v_and_b32_e32 v3, 0xfffff, v10 ; SDAG-NEXT: v_or3_b32 v1, v3, v1, v2 -; SDAG-NEXT: .LBB2_16: ; %Flow8 +; SDAG-NEXT: .LBB2_14: ; %Flow5 ; SDAG-NEXT: s_or_b64 exec, exec, s[6:7] ; SDAG-NEXT: s_setpc_b64 s[30:31] ; @@ -744,156 +668,142 @@ define double @sitofp_i128_to_f64(i128 %x) { ; GISEL-NEXT: v_mov_b32_e32 v0, s4 ; GISEL-NEXT: v_mov_b32_e32 v1, s5 ; GISEL-NEXT: s_and_saveexec_b64 s[6:7], vcc -; GISEL-NEXT: s_cbranch_execz .LBB2_16 +; GISEL-NEXT: s_cbranch_execz .LBB2_14 ; GISEL-NEXT: ; %bb.1: ; %itofp-if-end -; GISEL-NEXT: v_ashrrev_i32_e32 v8, 31, v3 -; GISEL-NEXT: v_xor_b32_e32 v0, v8, v4 -; GISEL-NEXT: v_xor_b32_e32 v1, v8, v5 -; GISEL-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v8 -; GISEL-NEXT: v_xor_b32_e32 v2, v8, v2 -; GISEL-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v8, vcc -; GISEL-NEXT: v_xor_b32_e32 v3, v8, v3 -; GISEL-NEXT: v_subb_co_u32_e32 v2, vcc, v2, v8, vcc +; GISEL-NEXT: v_ashrrev_i32_e32 v6, 31, v3 +; GISEL-NEXT: v_xor_b32_e32 v0, v6, v4 +; GISEL-NEXT: v_xor_b32_e32 v1, v6, v5 +; GISEL-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v6 +; GISEL-NEXT: v_xor_b32_e32 v2, v6, v2 +; GISEL-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v6, vcc +; GISEL-NEXT: v_xor_b32_e32 v3, v6, v3 +; GISEL-NEXT: v_subb_co_u32_e32 v2, vcc, v2, v6, vcc ; GISEL-NEXT: v_ffbh_u32_e32 v5, v0 -; GISEL-NEXT: v_subb_co_u32_e32 v3, vcc, v3, v8, vcc +; GISEL-NEXT: v_subb_co_u32_e32 v3, vcc, v3, v6, vcc ; GISEL-NEXT: v_ffbh_u32_e32 v4, v1 ; GISEL-NEXT: v_add_u32_e32 v5, 32, v5 -; GISEL-NEXT: v_ffbh_u32_e32 v6, v2 +; GISEL-NEXT: v_ffbh_u32_e32 v7, v2 ; GISEL-NEXT: v_min_u32_e32 v4, v4, v5 ; GISEL-NEXT: v_ffbh_u32_e32 v5, v3 -; GISEL-NEXT: v_add_u32_e32 v6, 32, v6 +; GISEL-NEXT: v_add_u32_e32 v7, 32, v7 ; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3] ; GISEL-NEXT: v_add_u32_e32 v4, 64, v4 -; GISEL-NEXT: v_min_u32_e32 v5, v5, v6 -; GISEL-NEXT: v_cndmask_b32_e32 v11, v5, v4, vcc -; GISEL-NEXT: v_sub_u32_e32 v9, 0x7f, v11 -; GISEL-NEXT: v_cmp_ge_i32_e32 vcc, 53, v9 -; GISEL-NEXT: ; implicit-def: $vgpr6 +; GISEL-NEXT: v_min_u32_e32 v5, v5, v7 +; GISEL-NEXT: v_cndmask_b32_e32 v9, v5, v4, vcc +; GISEL-NEXT: v_sub_u32_e32 v8, 0x80, v9 +; GISEL-NEXT: v_sub_u32_e32 v7, 0x7f, v9 +; GISEL-NEXT: v_cmp_ge_i32_e32 vcc, 53, v8 +; GISEL-NEXT: ; implicit-def: $vgpr10 ; GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GISEL-NEXT: ; %bb.2: ; %itofp-if-else -; GISEL-NEXT: v_add_u32_e32 v2, 0xffffffb5, v11 +; GISEL-NEXT: v_add_u32_e32 v2, 0xffffffb5, v9 ; GISEL-NEXT: v_lshlrev_b64 v[0:1], v2, v[0:1] ; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v2 ; GISEL-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc -; GISEL-NEXT: v_cndmask_b32_e32 v6, 0, v1, vcc +; GISEL-NEXT: v_cndmask_b32_e32 v10, 0, v1, vcc +; GISEL-NEXT: ; implicit-def: $vgpr8 ; GISEL-NEXT: ; implicit-def: $vgpr0 -; GISEL-NEXT: ; implicit-def: $vgpr11 -; GISEL-NEXT: ; %bb.3: ; %Flow6 +; GISEL-NEXT: ; implicit-def: $vgpr9 +; GISEL-NEXT: ; %bb.3: ; %Flow3 ; GISEL-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] -; GISEL-NEXT: s_cbranch_execz .LBB2_15 +; GISEL-NEXT: s_cbranch_execz .LBB2_13 ; GISEL-NEXT: ; %bb.4: ; %NodeBlock -; GISEL-NEXT: v_sub_u32_e32 v10, 0x80, v11 -; GISEL-NEXT: v_cmp_le_i32_e32 vcc, 55, v10 -; GISEL-NEXT: s_mov_b64 s[10:11], 0 -; GISEL-NEXT: s_mov_b64 s[4:5], 0 +; GISEL-NEXT: v_cmp_le_i32_e32 vcc, 55, v8 +; GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GISEL-NEXT: s_xor_b64 s[10:11], exec, s[4:5] +; GISEL-NEXT: s_cbranch_execz .LBB2_8 +; GISEL-NEXT: ; %bb.5: ; %LeafBlock +; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 55, v8 ; GISEL-NEXT: s_and_saveexec_b64 s[12:13], vcc -; GISEL-NEXT: s_xor_b64 s[12:13], exec, s[12:13] -; GISEL-NEXT: ; %bb.5: ; %LeafBlock1 -; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 55, v10 -; GISEL-NEXT: s_andn2_b64 s[4:5], 0, exec -; GISEL-NEXT: s_and_b64 s[14:15], exec, vcc -; GISEL-NEXT: s_or_b64 s[4:5], s[4:5], s[14:15] -; GISEL-NEXT: ; %bb.6: ; %Flow3 -; GISEL-NEXT: s_andn2_saveexec_b64 s[12:13], s[12:13] -; GISEL-NEXT: ; %bb.7: ; %LeafBlock -; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 54, v10 -; GISEL-NEXT: s_andn2_b64 s[10:11], 0, exec -; GISEL-NEXT: s_and_b64 s[14:15], exec, -1 -; GISEL-NEXT: s_or_b64 s[10:11], s[10:11], s[14:15] -; GISEL-NEXT: s_andn2_b64 s[4:5], s[4:5], exec -; GISEL-NEXT: s_and_b64 s[14:15], exec, vcc -; GISEL-NEXT: s_or_b64 s[4:5], s[4:5], s[14:15] -; GISEL-NEXT: ; %bb.8: ; %Flow4 -; GISEL-NEXT: s_or_b64 exec, exec, s[12:13] -; GISEL-NEXT: v_mov_b32_e32 v7, v3 -; GISEL-NEXT: v_mov_b32_e32 v6, v2 -; GISEL-NEXT: v_mov_b32_e32 v5, v1 -; GISEL-NEXT: v_mov_b32_e32 v4, v0 -; GISEL-NEXT: s_and_saveexec_b64 s[12:13], s[4:5] -; GISEL-NEXT: s_xor_b64 s[12:13], exec, s[12:13] -; GISEL-NEXT: s_cbranch_execz .LBB2_10 -; GISEL-NEXT: ; %bb.9: ; %itofp-sw-default -; GISEL-NEXT: v_sub_u32_e32 v14, 0x49, v11 -; GISEL-NEXT: v_sub_u32_e32 v6, 64, v14 +; GISEL-NEXT: s_cbranch_execz .LBB2_7 +; GISEL-NEXT: ; %bb.6: ; %itofp-sw-default +; GISEL-NEXT: v_sub_u32_e32 v14, 0x49, v9 +; GISEL-NEXT: v_sub_u32_e32 v10, 64, v14 ; GISEL-NEXT: v_lshrrev_b64 v[4:5], v14, v[0:1] -; GISEL-NEXT: v_lshlrev_b64 v[6:7], v6, v[2:3] +; GISEL-NEXT: v_lshlrev_b64 v[10:11], v10, v[2:3] ; GISEL-NEXT: v_subrev_u32_e32 v15, 64, v14 -; GISEL-NEXT: v_or_b32_e32 v6, v4, v6 -; GISEL-NEXT: v_or_b32_e32 v7, v5, v7 +; GISEL-NEXT: v_or_b32_e32 v10, v4, v10 +; GISEL-NEXT: v_or_b32_e32 v11, v5, v11 ; GISEL-NEXT: v_lshrrev_b64 v[4:5], v15, v[2:3] -; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v14 ; GISEL-NEXT: v_lshrrev_b64 v[12:13], v14, v[2:3] -; GISEL-NEXT: v_cndmask_b32_e32 v5, v5, v7, vcc -; GISEL-NEXT: v_add_u32_e32 v7, 55, v11 -; GISEL-NEXT: v_sub_u32_e32 v13, 64, v7 -; GISEL-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc +; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v14 ; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v14 -; GISEL-NEXT: v_cndmask_b32_e32 v6, 0, v12, vcc -; GISEL-NEXT: v_lshrrev_b64 v[11:12], v7, -1 -; GISEL-NEXT: v_lshlrev_b64 v[13:14], v13, -1 -; GISEL-NEXT: v_subrev_u32_e32 v15, 64, v7 -; GISEL-NEXT: v_or_b32_e32 v16, v11, v13 -; GISEL-NEXT: v_or_b32_e32 v17, v12, v14 -; GISEL-NEXT: v_lshrrev_b64 v[13:14], v15, -1 -; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v7 -; GISEL-NEXT: v_cndmask_b32_e64 v4, v4, v0, s[4:5] -; GISEL-NEXT: v_cndmask_b32_e64 v5, v5, v1, s[4:5] -; GISEL-NEXT: v_cndmask_b32_e32 v13, v13, v16, vcc -; GISEL-NEXT: v_cndmask_b32_e32 v14, v14, v17, vcc -; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v7 -; GISEL-NEXT: v_cndmask_b32_e32 v11, 0, v11, vcc -; GISEL-NEXT: v_cndmask_b32_e32 v12, 0, v12, vcc -; GISEL-NEXT: v_cndmask_b32_e64 v7, v13, -1, s[4:5] -; GISEL-NEXT: v_cndmask_b32_e64 v13, v14, -1, s[4:5] -; GISEL-NEXT: v_and_b32_e32 v11, v11, v2 -; GISEL-NEXT: v_and_b32_e32 v12, v12, v3 -; GISEL-NEXT: v_and_or_b32 v11, v7, v0, v11 -; GISEL-NEXT: v_and_or_b32 v12, v13, v1, v12 -; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[11:12] -; GISEL-NEXT: s_andn2_b64 s[4:5], s[10:11], exec -; GISEL-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc -; GISEL-NEXT: s_and_b64 s[10:11], exec, 0 -; GISEL-NEXT: v_or_b32_e32 v4, v4, v7 -; GISEL-NEXT: s_or_b64 s[10:11], s[4:5], s[10:11] -; GISEL-NEXT: .LBB2_10: ; %Flow5 +; GISEL-NEXT: v_add_u32_e32 v14, 55, v9 +; GISEL-NEXT: v_cndmask_b32_e32 v4, v4, v10, vcc +; GISEL-NEXT: v_cndmask_b32_e32 v5, v5, v11, vcc +; GISEL-NEXT: v_sub_u32_e32 v11, 64, v14 +; GISEL-NEXT: v_cndmask_b32_e64 v13, v4, v0, s[4:5] +; GISEL-NEXT: v_cndmask_b32_e64 v4, v5, v1, s[4:5] +; GISEL-NEXT: v_cndmask_b32_e32 v5, 0, v12, vcc +; GISEL-NEXT: v_lshrrev_b64 v[9:10], v14, -1 +; GISEL-NEXT: v_lshlrev_b64 v[11:12], v11, -1 +; GISEL-NEXT: v_subrev_u32_e32 v15, 64, v14 +; GISEL-NEXT: v_or_b32_e32 v16, v9, v11 +; GISEL-NEXT: v_or_b32_e32 v17, v10, v12 +; GISEL-NEXT: v_lshrrev_b64 v[11:12], v15, -1 +; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v14 +; GISEL-NEXT: v_cndmask_b32_e32 v11, v11, v16, vcc +; GISEL-NEXT: v_cndmask_b32_e32 v12, v12, v17, vcc +; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v14 +; GISEL-NEXT: v_cndmask_b32_e32 v9, 0, v9, vcc +; GISEL-NEXT: v_cndmask_b32_e32 v10, 0, v10, vcc +; GISEL-NEXT: v_cndmask_b32_e64 v11, v11, -1, s[4:5] +; GISEL-NEXT: v_cndmask_b32_e64 v12, v12, -1, s[4:5] +; GISEL-NEXT: v_and_b32_e32 v2, v9, v2 +; GISEL-NEXT: v_and_b32_e32 v3, v10, v3 +; GISEL-NEXT: v_and_or_b32 v0, v11, v0, v2 +; GISEL-NEXT: v_and_or_b32 v1, v12, v1, v3 +; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] +; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc +; GISEL-NEXT: v_or_b32_e32 v3, v13, v0 +; GISEL-NEXT: v_mov_b32_e32 v0, v3 +; GISEL-NEXT: v_mov_b32_e32 v1, v4 +; GISEL-NEXT: v_mov_b32_e32 v2, v5 +; GISEL-NEXT: v_mov_b32_e32 v3, v6 +; GISEL-NEXT: .LBB2_7: ; %Flow1 ; GISEL-NEXT: s_or_b64 exec, exec, s[12:13] -; GISEL-NEXT: s_and_saveexec_b64 s[4:5], s[10:11] -; GISEL-NEXT: ; %bb.11: ; %itofp-sw-bb +; GISEL-NEXT: .LBB2_8: ; %Flow2 +; GISEL-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11] +; GISEL-NEXT: ; %bb.9: ; %itofp-sw-bb +; GISEL-NEXT: v_lshlrev_b64 v[9:10], 1, v[0:1] ; GISEL-NEXT: v_lshlrev_b64 v[2:3], 1, v[2:3] -; GISEL-NEXT: v_lshlrev_b64 v[4:5], 1, v[0:1] ; GISEL-NEXT: v_lshrrev_b32_e32 v0, 31, v1 -; GISEL-NEXT: v_or_b32_e32 v6, v2, v0 -; GISEL-NEXT: ; %bb.12: ; %itofp-sw-epilog +; GISEL-NEXT: v_or_b32_e32 v11, v2, v0 +; GISEL-NEXT: v_mov_b32_e32 v0, v9 +; GISEL-NEXT: v_mov_b32_e32 v1, v10 +; GISEL-NEXT: v_mov_b32_e32 v2, v11 +; GISEL-NEXT: v_mov_b32_e32 v3, v12 +; GISEL-NEXT: ; %bb.10: ; %itofp-sw-epilog ; GISEL-NEXT: s_or_b64 exec, exec, s[4:5] -; GISEL-NEXT: v_bfe_u32 v0, v4, 2, 1 -; GISEL-NEXT: v_or_b32_e32 v0, v4, v0 +; GISEL-NEXT: v_bfe_u32 v3, v0, 2, 1 +; GISEL-NEXT: v_or_b32_e32 v0, v0, v3 ; GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 1, v0 -; GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v5, vcc -; GISEL-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v6, vcc +; GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc +; GISEL-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v2, vcc ; GISEL-NEXT: v_lshrrev_b64 v[4:5], 2, v[0:1] -; GISEL-NEXT: v_mov_b32_e32 v6, 0 -; GISEL-NEXT: v_and_b32_e32 v7, 0x800000, v1 -; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7] -; GISEL-NEXT: v_lshl_or_b32 v6, v2, 30, v5 +; GISEL-NEXT: v_mov_b32_e32 v9, 0 +; GISEL-NEXT: v_and_b32_e32 v10, 0x800000, v1 +; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[9:10] +; GISEL-NEXT: v_lshl_or_b32 v10, v2, 30, v5 ; GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc -; GISEL-NEXT: ; %bb.13: ; %itofp-if-then20 +; GISEL-NEXT: ; %bb.11: ; %itofp-if-then20 ; GISEL-NEXT: v_lshrrev_b64 v[4:5], 3, v[0:1] -; GISEL-NEXT: v_mov_b32_e32 v9, v10 -; GISEL-NEXT: v_lshl_or_b32 v6, v2, 29, v5 -; GISEL-NEXT: ; %bb.14: ; %Flow +; GISEL-NEXT: v_mov_b32_e32 v7, v8 +; GISEL-NEXT: v_lshl_or_b32 v10, v2, 29, v5 +; GISEL-NEXT: ; %bb.12: ; %Flow ; GISEL-NEXT: s_or_b64 exec, exec, s[4:5] -; GISEL-NEXT: .LBB2_15: ; %Flow7 +; GISEL-NEXT: .LBB2_13: ; %Flow4 ; GISEL-NEXT: s_or_b64 exec, exec, s[8:9] -; GISEL-NEXT: v_and_b32_e32 v0, 0x80000000, v8 +; GISEL-NEXT: v_and_b32_e32 v0, 0x80000000, v6 ; GISEL-NEXT: v_mov_b32_e32 v1, 0x3ff00000 ; GISEL-NEXT: v_mov_b32_e32 v2, 0xfffff -; GISEL-NEXT: v_lshl_add_u32 v1, v9, 20, v1 -; GISEL-NEXT: v_and_or_b32 v2, v6, v2, v0 +; GISEL-NEXT: v_lshl_add_u32 v1, v7, 20, v1 +; GISEL-NEXT: v_and_or_b32 v2, v10, v2, v0 ; GISEL-NEXT: v_and_or_b32 v0, v4, -1, 0 ; GISEL-NEXT: v_or3_b32 v1, v2, v1, 0 -; GISEL-NEXT: .LBB2_16: ; %Flow8 +; GISEL-NEXT: .LBB2_14: ; %Flow5 ; GISEL-NEXT: s_or_b64 exec, exec, s[6:7] ; GISEL-NEXT: s_setpc_b64 s[30:31] %cvt = sitofp i128 %x to double @@ -910,7 +820,7 @@ define double @uitofp_i128_to_f64(i128 %x) { ; SDAG-NEXT: v_mov_b32_e32 v4, 0 ; SDAG-NEXT: v_mov_b32_e32 v5, 0 ; SDAG-NEXT: s_and_saveexec_b64 s[6:7], vcc -; SDAG-NEXT: s_cbranch_execz .LBB3_16 +; SDAG-NEXT: s_cbranch_execz .LBB3_14 ; SDAG-NEXT: ; %bb.1: ; %itofp-if-end ; SDAG-NEXT: v_ffbh_u32_e32 v4, v2 ; SDAG-NEXT: v_add_u32_e32 v4, 32, v4 @@ -922,128 +832,112 @@ define double @uitofp_i128_to_f64(i128 %x) { ; SDAG-NEXT: v_min_u32_e32 v5, v5, v6 ; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3] ; SDAG-NEXT: v_add_u32_e32 v5, 64, v5 -; SDAG-NEXT: v_cndmask_b32_e32 v11, v5, v4, vcc +; SDAG-NEXT: v_cndmask_b32_e32 v8, v5, v4, vcc +; SDAG-NEXT: v_sub_u32_e32 v7, 0x80, v8 +; SDAG-NEXT: v_sub_u32_e32 v6, 0x7f, v8 +; SDAG-NEXT: v_cmp_gt_i32_e32 vcc, 54, v7 +; SDAG-NEXT: ; implicit-def: $vgpr9 ; SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5 -; SDAG-NEXT: v_sub_u32_e32 v9, 0x7f, v11 -; SDAG-NEXT: v_mov_b32_e32 v6, v1 -; SDAG-NEXT: v_mov_b32_e32 v8, v3 -; SDAG-NEXT: v_cmp_gt_i32_e32 vcc, 54, v9 -; SDAG-NEXT: v_mov_b32_e32 v5, v0 -; SDAG-NEXT: v_mov_b32_e32 v7, v2 -; SDAG-NEXT: ; implicit-def: $vgpr12 ; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc ; SDAG-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; SDAG-NEXT: ; %bb.2: ; %itofp-if-else -; SDAG-NEXT: v_add_u32_e32 v2, 0xffffffb5, v11 +; SDAG-NEXT: v_add_u32_e32 v2, 0xffffffb5, v8 ; SDAG-NEXT: v_lshlrev_b64 v[0:1], v2, v[0:1] ; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v2 -; SDAG-NEXT: v_cndmask_b32_e32 v12, 0, v1, vcc +; SDAG-NEXT: v_cndmask_b32_e32 v9, 0, v1, vcc ; SDAG-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc +; SDAG-NEXT: ; implicit-def: $vgpr7 ; SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3 ; SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 -; SDAG-NEXT: ; implicit-def: $vgpr11 -; SDAG-NEXT: ; implicit-def: $vgpr5_vgpr6 -; SDAG-NEXT: ; implicit-def: $vgpr7_vgpr8 -; SDAG-NEXT: ; %bb.3: ; %Flow6 +; SDAG-NEXT: ; implicit-def: $vgpr8 +; SDAG-NEXT: ; %bb.3: ; %Flow3 ; SDAG-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] -; SDAG-NEXT: s_cbranch_execz .LBB3_15 +; SDAG-NEXT: s_cbranch_execz .LBB3_13 ; SDAG-NEXT: ; %bb.4: ; %NodeBlock -; SDAG-NEXT: v_sub_u32_e32 v10, 0x80, v11 -; SDAG-NEXT: v_cmp_lt_i32_e32 vcc, 54, v10 -; SDAG-NEXT: s_mov_b64 s[10:11], 0 -; SDAG-NEXT: s_mov_b64 s[4:5], 0 +; SDAG-NEXT: v_cmp_lt_i32_e32 vcc, 54, v7 +; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc +; SDAG-NEXT: s_xor_b64 s[10:11], exec, s[4:5] +; SDAG-NEXT: s_cbranch_execz .LBB3_8 +; SDAG-NEXT: ; %bb.5: ; %LeafBlock +; SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 55, v7 ; SDAG-NEXT: s_and_saveexec_b64 s[12:13], vcc -; SDAG-NEXT: s_xor_b64 s[12:13], exec, s[12:13] -; SDAG-NEXT: ; %bb.5: ; %LeafBlock1 -; SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 55, v10 -; SDAG-NEXT: s_and_b64 s[4:5], vcc, exec -; SDAG-NEXT: ; %bb.6: ; %Flow3 -; SDAG-NEXT: s_andn2_saveexec_b64 s[12:13], s[12:13] -; SDAG-NEXT: ; %bb.7: ; %LeafBlock -; SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 54, v10 -; SDAG-NEXT: s_andn2_b64 s[4:5], s[4:5], exec -; SDAG-NEXT: s_and_b64 s[14:15], vcc, exec -; SDAG-NEXT: s_mov_b64 s[10:11], exec -; SDAG-NEXT: s_or_b64 s[4:5], s[4:5], s[14:15] -; SDAG-NEXT: ; implicit-def: $vgpr5_vgpr6 -; SDAG-NEXT: ; implicit-def: $vgpr7_vgpr8 -; SDAG-NEXT: ; %bb.8: ; %Flow4 -; SDAG-NEXT: s_or_b64 exec, exec, s[12:13] -; SDAG-NEXT: s_and_saveexec_b64 s[12:13], s[4:5] -; SDAG-NEXT: s_xor_b64 s[12:13], exec, s[12:13] -; SDAG-NEXT: s_cbranch_execz .LBB3_10 -; SDAG-NEXT: ; %bb.9: ; %itofp-sw-default -; SDAG-NEXT: v_sub_u32_e32 v8, 0x49, v11 -; SDAG-NEXT: v_sub_u32_e32 v6, 64, v8 -; SDAG-NEXT: v_lshrrev_b64 v[4:5], v8, v[0:1] -; SDAG-NEXT: v_lshlrev_b64 v[6:7], v6, v[2:3] -; SDAG-NEXT: v_sub_u32_e32 v13, 9, v11 -; SDAG-NEXT: v_or_b32_e32 v7, v5, v7 -; SDAG-NEXT: v_or_b32_e32 v12, v4, v6 -; SDAG-NEXT: v_lshrrev_b64 v[4:5], v13, v[2:3] -; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v8 -; SDAG-NEXT: v_cndmask_b32_e32 v5, v5, v7, vcc -; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v8 -; SDAG-NEXT: v_cndmask_b32_e64 v6, v5, v1, s[4:5] -; SDAG-NEXT: v_cndmask_b32_e32 v7, v4, v12, vcc -; SDAG-NEXT: v_lshrrev_b64 v[4:5], v8, v[2:3] -; SDAG-NEXT: v_add_u32_e32 v16, 55, v11 -; SDAG-NEXT: v_cndmask_b32_e64 v8, v7, v0, s[4:5] -; SDAG-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc -; SDAG-NEXT: v_lshrrev_b64 v[12:13], v13, v[0:1] -; SDAG-NEXT: v_lshlrev_b64 v[14:15], v16, v[2:3] -; SDAG-NEXT: v_cndmask_b32_e32 v7, 0, v4, vcc -; SDAG-NEXT: v_add_u32_e32 v4, -9, v11 -; SDAG-NEXT: v_lshlrev_b64 v[4:5], v4, v[0:1] -; SDAG-NEXT: v_or_b32_e32 v13, v15, v13 -; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v16 -; SDAG-NEXT: v_or_b32_e32 v12, v14, v12 -; SDAG-NEXT: v_cndmask_b32_e32 v5, v5, v13, vcc -; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v16 -; SDAG-NEXT: v_cndmask_b32_e64 v11, v5, v3, s[4:5] -; SDAG-NEXT: v_cndmask_b32_e32 v12, v4, v12, vcc -; SDAG-NEXT: v_lshlrev_b64 v[4:5], v16, v[0:1] -; SDAG-NEXT: v_cndmask_b32_e64 v12, v12, v2, s[4:5] -; SDAG-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc -; SDAG-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc -; SDAG-NEXT: v_or_b32_e32 v5, v5, v11 -; SDAG-NEXT: v_or_b32_e32 v4, v4, v12 -; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] -; SDAG-NEXT: s_andn2_b64 s[10:11], s[10:11], exec -; SDAG-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc -; SDAG-NEXT: v_or_b32_e32 v5, v8, v4 -; SDAG-NEXT: .LBB3_10: ; %Flow5 +; SDAG-NEXT: s_cbranch_execz .LBB3_7 +; SDAG-NEXT: ; %bb.6: ; %itofp-sw-default +; SDAG-NEXT: v_sub_u32_e32 v11, 0x49, v8 +; SDAG-NEXT: v_sub_u32_e32 v9, 64, v11 +; SDAG-NEXT: v_lshrrev_b64 v[4:5], v11, v[0:1] +; SDAG-NEXT: v_lshlrev_b64 v[9:10], v9, v[2:3] +; SDAG-NEXT: v_sub_u32_e32 v12, 9, v8 +; SDAG-NEXT: v_or_b32_e32 v10, v5, v10 +; SDAG-NEXT: v_or_b32_e32 v9, v4, v9 +; SDAG-NEXT: v_lshrrev_b64 v[4:5], v12, v[2:3] +; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v11 +; SDAG-NEXT: v_add_u32_e32 v15, 55, v8 +; SDAG-NEXT: v_cndmask_b32_e32 v5, v5, v10, vcc +; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v11 +; SDAG-NEXT: v_cndmask_b32_e32 v4, v4, v9, vcc +; SDAG-NEXT: v_lshrrev_b64 v[9:10], v11, v[2:3] +; SDAG-NEXT: v_lshrrev_b64 v[11:12], v12, v[0:1] +; SDAG-NEXT: v_lshlrev_b64 v[13:14], v15, v[2:3] +; SDAG-NEXT: v_add_u32_e32 v8, -9, v8 +; SDAG-NEXT: v_or_b32_e32 v14, v14, v12 +; SDAG-NEXT: v_or_b32_e32 v13, v13, v11 +; SDAG-NEXT: v_lshlrev_b64 v[11:12], v8, v[0:1] +; SDAG-NEXT: v_cndmask_b32_e32 v10, 0, v10, vcc +; SDAG-NEXT: v_cndmask_b32_e32 v9, 0, v9, vcc +; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v15 +; SDAG-NEXT: v_cndmask_b32_e64 v5, v5, v1, s[4:5] +; SDAG-NEXT: v_cndmask_b32_e64 v4, v4, v0, s[4:5] +; SDAG-NEXT: v_cndmask_b32_e32 v8, v12, v14, vcc +; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v15 +; SDAG-NEXT: v_lshlrev_b64 v[0:1], v15, v[0:1] +; SDAG-NEXT: v_cndmask_b32_e64 v3, v8, v3, s[4:5] +; SDAG-NEXT: v_cndmask_b32_e32 v8, v11, v13, vcc +; SDAG-NEXT: v_cndmask_b32_e64 v2, v8, v2, s[4:5] +; SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc +; SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc +; SDAG-NEXT: v_or_b32_e32 v1, v1, v3 +; SDAG-NEXT: v_or_b32_e32 v0, v0, v2 +; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] +; SDAG-NEXT: v_mov_b32_e32 v2, v9 +; SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc +; SDAG-NEXT: v_or_b32_e32 v4, v4, v0 +; SDAG-NEXT: v_mov_b32_e32 v0, v4 +; SDAG-NEXT: v_mov_b32_e32 v1, v5 +; SDAG-NEXT: v_mov_b32_e32 v3, v10 +; SDAG-NEXT: .LBB3_7: ; %Flow1 ; SDAG-NEXT: s_or_b64 exec, exec, s[12:13] -; SDAG-NEXT: s_and_saveexec_b64 s[4:5], s[10:11] -; SDAG-NEXT: ; %bb.11: ; %itofp-sw-bb -; SDAG-NEXT: v_lshlrev_b64 v[7:8], 1, v[2:3] -; SDAG-NEXT: v_lshrrev_b32_e32 v2, 31, v1 -; SDAG-NEXT: v_lshlrev_b64 v[5:6], 1, v[0:1] -; SDAG-NEXT: v_or_b32_e32 v7, v7, v2 -; SDAG-NEXT: ; %bb.12: ; %itofp-sw-epilog +; SDAG-NEXT: .LBB3_8: ; %Flow2 +; SDAG-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11] +; SDAG-NEXT: ; %bb.9: ; %itofp-sw-bb +; SDAG-NEXT: v_lshlrev_b64 v[2:3], 1, v[2:3] +; SDAG-NEXT: v_lshrrev_b32_e32 v3, 31, v1 +; SDAG-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1] +; SDAG-NEXT: v_or_b32_e32 v2, v2, v3 +; SDAG-NEXT: ; %bb.10: ; %itofp-sw-epilog ; SDAG-NEXT: s_or_b64 exec, exec, s[4:5] -; SDAG-NEXT: v_lshrrev_b32_e32 v0, 2, v5 -; SDAG-NEXT: v_and_or_b32 v0, v0, 1, v5 +; SDAG-NEXT: v_lshrrev_b32_e32 v3, 2, v0 +; SDAG-NEXT: v_and_or_b32 v0, v3, 1, v0 ; SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 1, v0 -; SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v6, vcc -; SDAG-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v7, vcc +; SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc +; SDAG-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v2, vcc ; SDAG-NEXT: v_lshrrev_b64 v[4:5], 2, v[0:1] ; SDAG-NEXT: v_and_b32_e32 v3, 0x800000, v1 ; SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 0, v3 -; SDAG-NEXT: v_alignbit_b32 v12, v2, v1, 2 +; SDAG-NEXT: v_alignbit_b32 v9, v2, v1, 2 ; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc -; SDAG-NEXT: ; %bb.13: ; %itofp-if-then20 +; SDAG-NEXT: ; %bb.11: ; %itofp-if-then20 ; SDAG-NEXT: v_lshrrev_b64 v[4:5], 3, v[0:1] -; SDAG-NEXT: v_alignbit_b32 v12, v2, v1, 3 -; SDAG-NEXT: v_mov_b32_e32 v9, v10 -; SDAG-NEXT: ; %bb.14: ; %Flow +; SDAG-NEXT: v_alignbit_b32 v9, v2, v1, 3 +; SDAG-NEXT: v_mov_b32_e32 v6, v7 +; SDAG-NEXT: ; %bb.12: ; %Flow ; SDAG-NEXT: s_or_b64 exec, exec, s[4:5] -; SDAG-NEXT: .LBB3_15: ; %Flow7 +; SDAG-NEXT: .LBB3_13: ; %Flow4 ; SDAG-NEXT: s_or_b64 exec, exec, s[8:9] -; SDAG-NEXT: v_and_b32_e32 v0, 0xfffff, v12 -; SDAG-NEXT: v_lshl_or_b32 v0, v9, 20, v0 +; SDAG-NEXT: v_and_b32_e32 v0, 0xfffff, v9 +; SDAG-NEXT: v_lshl_or_b32 v0, v6, 20, v0 ; SDAG-NEXT: v_add_u32_e32 v5, 0x3ff00000, v0 -; SDAG-NEXT: .LBB3_16: ; %Flow8 +; SDAG-NEXT: .LBB3_14: ; %Flow5 ; SDAG-NEXT: s_or_b64 exec, exec, s[6:7] ; SDAG-NEXT: v_mov_b32_e32 v0, v4 ; SDAG-NEXT: v_mov_b32_e32 v1, v5 @@ -1059,7 +953,7 @@ define double @uitofp_i128_to_f64(i128 %x) { ; GISEL-NEXT: v_mov_b32_e32 v4, s4 ; GISEL-NEXT: v_mov_b32_e32 v5, s5 ; GISEL-NEXT: s_and_saveexec_b64 s[6:7], vcc -; GISEL-NEXT: s_cbranch_execz .LBB3_16 +; GISEL-NEXT: s_cbranch_execz .LBB3_14 ; GISEL-NEXT: ; %bb.1: ; %itofp-if-end ; GISEL-NEXT: v_ffbh_u32_e32 v5, v0 ; GISEL-NEXT: v_ffbh_u32_e32 v4, v1 @@ -1071,139 +965,125 @@ define double @uitofp_i128_to_f64(i128 %x) { ; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3] ; GISEL-NEXT: v_add_u32_e32 v4, 64, v4 ; GISEL-NEXT: v_min_u32_e32 v5, v5, v6 -; GISEL-NEXT: v_cndmask_b32_e32 v10, v5, v4, vcc -; GISEL-NEXT: v_sub_u32_e32 v8, 0x7f, v10 -; GISEL-NEXT: v_cmp_ge_i32_e32 vcc, 53, v8 -; GISEL-NEXT: ; implicit-def: $vgpr6 +; GISEL-NEXT: v_cndmask_b32_e32 v8, v5, v4, vcc +; GISEL-NEXT: v_sub_u32_e32 v7, 0x80, v8 +; GISEL-NEXT: v_sub_u32_e32 v6, 0x7f, v8 +; GISEL-NEXT: v_cmp_ge_i32_e32 vcc, 53, v7 +; GISEL-NEXT: ; implicit-def: $vgpr9 ; GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5 ; GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GISEL-NEXT: ; %bb.2: ; %itofp-if-else -; GISEL-NEXT: v_add_u32_e32 v2, 0xffffffb5, v10 +; GISEL-NEXT: v_add_u32_e32 v2, 0xffffffb5, v8 ; GISEL-NEXT: v_lshlrev_b64 v[0:1], v2, v[0:1] ; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v2 ; GISEL-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc -; GISEL-NEXT: v_cndmask_b32_e32 v6, 0, v1, vcc +; GISEL-NEXT: v_cndmask_b32_e32 v9, 0, v1, vcc +; GISEL-NEXT: ; implicit-def: $vgpr7 ; GISEL-NEXT: ; implicit-def: $vgpr0 -; GISEL-NEXT: ; implicit-def: $vgpr10 -; GISEL-NEXT: ; %bb.3: ; %Flow6 +; GISEL-NEXT: ; implicit-def: $vgpr8 +; GISEL-NEXT: ; %bb.3: ; %Flow3 ; GISEL-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] -; GISEL-NEXT: s_cbranch_execz .LBB3_15 +; GISEL-NEXT: s_cbranch_execz .LBB3_13 ; GISEL-NEXT: ; %bb.4: ; %NodeBlock -; GISEL-NEXT: v_sub_u32_e32 v9, 0x80, v10 -; GISEL-NEXT: v_cmp_le_i32_e32 vcc, 55, v9 -; GISEL-NEXT: s_mov_b64 s[10:11], 0 -; GISEL-NEXT: s_mov_b64 s[4:5], 0 +; GISEL-NEXT: v_cmp_le_i32_e32 vcc, 55, v7 +; GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GISEL-NEXT: s_xor_b64 s[10:11], exec, s[4:5] +; GISEL-NEXT: s_cbranch_execz .LBB3_8 +; GISEL-NEXT: ; %bb.5: ; %LeafBlock +; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 55, v7 ; GISEL-NEXT: s_and_saveexec_b64 s[12:13], vcc -; GISEL-NEXT: s_xor_b64 s[12:13], exec, s[12:13] -; GISEL-NEXT: ; %bb.5: ; %LeafBlock1 -; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 55, v9 -; GISEL-NEXT: s_andn2_b64 s[4:5], 0, exec -; GISEL-NEXT: s_and_b64 s[14:15], exec, vcc -; GISEL-NEXT: s_or_b64 s[4:5], s[4:5], s[14:15] -; GISEL-NEXT: ; %bb.6: ; %Flow3 -; GISEL-NEXT: s_andn2_saveexec_b64 s[12:13], s[12:13] -; GISEL-NEXT: ; %bb.7: ; %LeafBlock -; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 54, v9 -; GISEL-NEXT: s_andn2_b64 s[10:11], 0, exec -; GISEL-NEXT: s_and_b64 s[14:15], exec, -1 -; GISEL-NEXT: s_or_b64 s[10:11], s[10:11], s[14:15] -; GISEL-NEXT: s_andn2_b64 s[4:5], s[4:5], exec -; GISEL-NEXT: s_and_b64 s[14:15], exec, vcc -; GISEL-NEXT: s_or_b64 s[4:5], s[4:5], s[14:15] -; GISEL-NEXT: ; %bb.8: ; %Flow4 -; GISEL-NEXT: s_or_b64 exec, exec, s[12:13] -; GISEL-NEXT: v_mov_b32_e32 v7, v3 -; GISEL-NEXT: v_mov_b32_e32 v6, v2 -; GISEL-NEXT: v_mov_b32_e32 v5, v1 -; GISEL-NEXT: v_mov_b32_e32 v4, v0 -; GISEL-NEXT: s_and_saveexec_b64 s[12:13], s[4:5] -; GISEL-NEXT: s_xor_b64 s[12:13], exec, s[12:13] -; GISEL-NEXT: s_cbranch_execz .LBB3_10 -; GISEL-NEXT: ; %bb.9: ; %itofp-sw-default -; GISEL-NEXT: v_sub_u32_e32 v13, 0x49, v10 -; GISEL-NEXT: v_sub_u32_e32 v6, 64, v13 +; GISEL-NEXT: s_cbranch_execz .LBB3_7 +; GISEL-NEXT: ; %bb.6: ; %itofp-sw-default +; GISEL-NEXT: v_sub_u32_e32 v13, 0x49, v8 +; GISEL-NEXT: v_sub_u32_e32 v9, 64, v13 ; GISEL-NEXT: v_lshrrev_b64 v[4:5], v13, v[0:1] -; GISEL-NEXT: v_lshlrev_b64 v[6:7], v6, v[2:3] +; GISEL-NEXT: v_lshlrev_b64 v[9:10], v9, v[2:3] ; GISEL-NEXT: v_subrev_u32_e32 v14, 64, v13 ; GISEL-NEXT: v_lshrrev_b64 v[11:12], v13, v[2:3] -; GISEL-NEXT: v_or_b32_e32 v6, v4, v6 -; GISEL-NEXT: v_or_b32_e32 v7, v5, v7 +; GISEL-NEXT: v_or_b32_e32 v9, v4, v9 +; GISEL-NEXT: v_or_b32_e32 v10, v5, v10 ; GISEL-NEXT: v_lshrrev_b64 v[4:5], v14, v[2:3] ; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v13 -; GISEL-NEXT: v_add_u32_e32 v14, 55, v10 -; GISEL-NEXT: v_cndmask_b32_e32 v5, v5, v7, vcc -; GISEL-NEXT: v_cndmask_b32_e32 v7, 0, v12, vcc -; GISEL-NEXT: v_sub_u32_e32 v12, 64, v14 -; GISEL-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc +; GISEL-NEXT: v_add_u32_e32 v8, 55, v8 +; GISEL-NEXT: v_cndmask_b32_e32 v4, v4, v9, vcc +; GISEL-NEXT: v_cndmask_b32_e32 v5, v5, v10, vcc ; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v13 -; GISEL-NEXT: v_cndmask_b32_e32 v6, 0, v11, vcc -; GISEL-NEXT: v_lshrrev_b64 v[10:11], v14, -1 +; GISEL-NEXT: v_cndmask_b32_e32 v10, 0, v11, vcc +; GISEL-NEXT: v_cndmask_b32_e32 v11, 0, v12, vcc +; GISEL-NEXT: v_sub_u32_e32 v12, 64, v8 +; GISEL-NEXT: v_cndmask_b32_e64 v14, v4, v0, s[4:5] +; GISEL-NEXT: v_cndmask_b32_e64 v9, v5, v1, s[4:5] +; GISEL-NEXT: v_lshrrev_b64 v[4:5], v8, -1 ; GISEL-NEXT: v_lshlrev_b64 v[12:13], v12, -1 -; GISEL-NEXT: v_subrev_u32_e32 v15, 64, v14 -; GISEL-NEXT: v_or_b32_e32 v16, v10, v12 -; GISEL-NEXT: v_or_b32_e32 v17, v11, v13 +; GISEL-NEXT: v_subrev_u32_e32 v15, 64, v8 +; GISEL-NEXT: v_or_b32_e32 v16, v4, v12 +; GISEL-NEXT: v_or_b32_e32 v17, v5, v13 ; GISEL-NEXT: v_lshrrev_b64 v[12:13], v15, -1 -; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v14 -; GISEL-NEXT: v_cndmask_b32_e64 v4, v4, v0, s[4:5] -; GISEL-NEXT: v_cndmask_b32_e64 v5, v5, v1, s[4:5] +; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v8 ; GISEL-NEXT: v_cndmask_b32_e32 v12, v12, v16, vcc ; GISEL-NEXT: v_cndmask_b32_e32 v13, v13, v17, vcc -; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v14 -; GISEL-NEXT: v_cndmask_b32_e32 v10, 0, v10, vcc -; GISEL-NEXT: v_cndmask_b32_e32 v11, 0, v11, vcc -; GISEL-NEXT: v_cndmask_b32_e64 v12, v12, -1, s[4:5] -; GISEL-NEXT: v_cndmask_b32_e64 v13, v13, -1, s[4:5] -; GISEL-NEXT: v_and_b32_e32 v10, v10, v2 -; GISEL-NEXT: v_and_b32_e32 v11, v11, v3 -; GISEL-NEXT: v_and_or_b32 v10, v12, v0, v10 -; GISEL-NEXT: v_and_or_b32 v11, v13, v1, v11 -; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11] -; GISEL-NEXT: s_andn2_b64 s[4:5], s[10:11], exec -; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc -; GISEL-NEXT: s_and_b64 s[10:11], exec, 0 -; GISEL-NEXT: v_or_b32_e32 v4, v4, v10 -; GISEL-NEXT: s_or_b64 s[10:11], s[4:5], s[10:11] -; GISEL-NEXT: .LBB3_10: ; %Flow5 +; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v8 +; GISEL-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc +; GISEL-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc +; GISEL-NEXT: v_cndmask_b32_e64 v8, v12, -1, s[4:5] +; GISEL-NEXT: v_cndmask_b32_e64 v12, v13, -1, s[4:5] +; GISEL-NEXT: v_and_b32_e32 v2, v4, v2 +; GISEL-NEXT: v_and_b32_e32 v3, v5, v3 +; GISEL-NEXT: v_and_or_b32 v0, v8, v0, v2 +; GISEL-NEXT: v_and_or_b32 v1, v12, v1, v3 +; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] +; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc +; GISEL-NEXT: v_or_b32_e32 v8, v14, v0 +; GISEL-NEXT: v_mov_b32_e32 v0, v8 +; GISEL-NEXT: v_mov_b32_e32 v1, v9 +; GISEL-NEXT: v_mov_b32_e32 v2, v10 +; GISEL-NEXT: v_mov_b32_e32 v3, v11 +; GISEL-NEXT: .LBB3_7: ; %Flow1 ; GISEL-NEXT: s_or_b64 exec, exec, s[12:13] -; GISEL-NEXT: s_and_saveexec_b64 s[4:5], s[10:11] -; GISEL-NEXT: ; %bb.11: ; %itofp-sw-bb -; GISEL-NEXT: v_lshlrev_b64 v[6:7], 1, v[2:3] -; GISEL-NEXT: v_lshlrev_b64 v[4:5], 1, v[0:1] +; GISEL-NEXT: .LBB3_8: ; %Flow2 +; GISEL-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11] +; GISEL-NEXT: ; %bb.9: ; %itofp-sw-bb +; GISEL-NEXT: v_lshlrev_b64 v[8:9], 1, v[0:1] +; GISEL-NEXT: v_lshlrev_b64 v[10:11], 1, v[2:3] ; GISEL-NEXT: v_lshrrev_b32_e32 v0, 31, v1 -; GISEL-NEXT: v_or_b32_e32 v6, v6, v0 -; GISEL-NEXT: ; %bb.12: ; %itofp-sw-epilog +; GISEL-NEXT: v_or_b32_e32 v10, v10, v0 +; GISEL-NEXT: v_mov_b32_e32 v0, v8 +; GISEL-NEXT: v_mov_b32_e32 v1, v9 +; GISEL-NEXT: v_mov_b32_e32 v2, v10 +; GISEL-NEXT: v_mov_b32_e32 v3, v11 +; GISEL-NEXT: ; %bb.10: ; %itofp-sw-epilog ; GISEL-NEXT: s_or_b64 exec, exec, s[4:5] -; GISEL-NEXT: v_bfe_u32 v0, v4, 2, 1 -; GISEL-NEXT: v_or_b32_e32 v0, v4, v0 +; GISEL-NEXT: v_bfe_u32 v4, v0, 2, 1 +; GISEL-NEXT: v_or_b32_e32 v0, v0, v4 ; GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 1, v0 -; GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v5, vcc -; GISEL-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v6, vcc +; GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc +; GISEL-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v2, vcc +; GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc +; GISEL-NEXT: v_mov_b32_e32 v8, 0 +; GISEL-NEXT: v_and_b32_e32 v9, 0x800000, v1 ; GISEL-NEXT: v_lshrrev_b64 v[4:5], 2, v[0:1] -; GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v7, vcc -; GISEL-NEXT: v_mov_b32_e32 v5, 0 -; GISEL-NEXT: v_and_b32_e32 v6, 0x800000, v1 -; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[5:6] -; GISEL-NEXT: v_lshlrev_b64 v[5:6], 30, v[2:3] -; GISEL-NEXT: v_lshrrev_b32_e32 v6, 2, v1 -; GISEL-NEXT: v_or_b32_e32 v6, v6, v5 +; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9] +; GISEL-NEXT: v_lshlrev_b64 v[8:9], 30, v[2:3] +; GISEL-NEXT: v_lshrrev_b32_e32 v5, 2, v1 +; GISEL-NEXT: v_or_b32_e32 v9, v5, v8 ; GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc -; GISEL-NEXT: ; %bb.13: ; %itofp-if-then20 +; GISEL-NEXT: ; %bb.11: ; %itofp-if-then20 ; GISEL-NEXT: v_lshlrev_b64 v[2:3], 29, v[2:3] ; GISEL-NEXT: v_lshrrev_b64 v[4:5], 3, v[0:1] ; GISEL-NEXT: v_lshrrev_b32_e32 v0, 3, v1 -; GISEL-NEXT: v_or_b32_e32 v6, v0, v2 -; GISEL-NEXT: v_mov_b32_e32 v8, v9 -; GISEL-NEXT: ; %bb.14: ; %Flow +; GISEL-NEXT: v_or_b32_e32 v9, v0, v2 +; GISEL-NEXT: v_mov_b32_e32 v6, v7 +; GISEL-NEXT: ; %bb.12: ; %Flow ; GISEL-NEXT: s_or_b64 exec, exec, s[4:5] -; GISEL-NEXT: .LBB3_15: ; %Flow7 +; GISEL-NEXT: .LBB3_13: ; %Flow4 ; GISEL-NEXT: s_or_b64 exec, exec, s[8:9] ; GISEL-NEXT: v_mov_b32_e32 v0, 0x3ff00000 -; GISEL-NEXT: v_lshl_add_u32 v0, v8, 20, v0 -; GISEL-NEXT: v_and_b32_e32 v1, 0xfffff, v6 +; GISEL-NEXT: v_lshl_add_u32 v0, v6, 20, v0 +; GISEL-NEXT: v_and_b32_e32 v1, 0xfffff, v9 ; GISEL-NEXT: v_and_or_b32 v4, v4, -1, 0 ; GISEL-NEXT: v_or3_b32 v5, v1, v0, 0 -; GISEL-NEXT: .LBB3_16: ; %Flow8 +; GISEL-NEXT: .LBB3_14: ; %Flow5 ; GISEL-NEXT: s_or_b64 exec, exec, s[6:7] ; GISEL-NEXT: v_mov_b32_e32 v0, v4 ; GISEL-NEXT: v_mov_b32_e32 v1, v5 @@ -1221,7 +1101,7 @@ define half @sitofp_i128_to_f16(i128 %x) { ; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; SDAG-NEXT: v_mov_b32_e32 v4, 0 ; SDAG-NEXT: s_and_saveexec_b64 s[6:7], vcc -; SDAG-NEXT: s_cbranch_execz .LBB4_16 +; SDAG-NEXT: s_cbranch_execz .LBB4_14 ; SDAG-NEXT: ; %bb.1: ; %itofp-if-end ; SDAG-NEXT: v_ashrrev_i32_e32 v5, 31, v3 ; SDAG-NEXT: v_xor_b32_e32 v0, v5, v0 @@ -1242,113 +1122,101 @@ define half @sitofp_i128_to_f16(i128 %x) { ; SDAG-NEXT: v_min_u32_e32 v6, v6, v7 ; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; SDAG-NEXT: v_add_u32_e32 v6, 64, v6 -; SDAG-NEXT: v_cndmask_b32_e32 v9, v6, v2, vcc -; SDAG-NEXT: v_sub_u32_e32 v2, 0x7f, v9 -; SDAG-NEXT: v_cmp_gt_i32_e32 vcc, 25, v2 -; SDAG-NEXT: ; implicit-def: $vgpr6 +; SDAG-NEXT: v_cndmask_b32_e32 v7, v6, v2, vcc +; SDAG-NEXT: v_sub_u32_e32 v6, 0x80, v7 +; SDAG-NEXT: v_sub_u32_e32 v2, 0x7f, v7 +; SDAG-NEXT: v_cmp_gt_i32_e32 vcc, 25, v6 +; SDAG-NEXT: ; implicit-def: $vgpr8 ; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc ; SDAG-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; SDAG-NEXT: ; %bb.2: ; %itofp-if-else -; SDAG-NEXT: v_add_u32_e32 v4, 0xffffff98, v9 +; SDAG-NEXT: v_add_u32_e32 v4, 0xffffff98, v7 ; SDAG-NEXT: v_lshlrev_b64 v[0:1], v4, v[0:1] ; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v4 -; SDAG-NEXT: v_cndmask_b32_e32 v6, 0, v0, vcc +; SDAG-NEXT: v_cndmask_b32_e32 v8, 0, v0, vcc +; SDAG-NEXT: ; implicit-def: $vgpr6 ; SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 -; SDAG-NEXT: ; implicit-def: $vgpr9 +; SDAG-NEXT: ; implicit-def: $vgpr7 ; SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5 -; SDAG-NEXT: ; %bb.3: ; %Flow6 +; SDAG-NEXT: ; %bb.3: ; %Flow3 ; SDAG-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] -; SDAG-NEXT: s_cbranch_execz .LBB4_15 +; SDAG-NEXT: s_cbranch_execz .LBB4_13 ; SDAG-NEXT: ; %bb.4: ; %NodeBlock -; SDAG-NEXT: v_sub_u32_e32 v8, 0x80, v9 -; SDAG-NEXT: v_cmp_lt_i32_e32 vcc, 25, v8 -; SDAG-NEXT: s_mov_b64 s[10:11], 0 -; SDAG-NEXT: s_mov_b64 s[4:5], 0 +; SDAG-NEXT: v_cmp_lt_i32_e32 vcc, 25, v6 +; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc +; SDAG-NEXT: s_xor_b64 s[10:11], exec, s[4:5] +; SDAG-NEXT: s_cbranch_execz .LBB4_8 +; SDAG-NEXT: ; %bb.5: ; %LeafBlock +; SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 26, v6 ; SDAG-NEXT: s_and_saveexec_b64 s[12:13], vcc -; SDAG-NEXT: s_xor_b64 s[12:13], exec, s[12:13] -; SDAG-NEXT: ; %bb.5: ; %LeafBlock1 -; SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 26, v8 -; SDAG-NEXT: s_and_b64 s[4:5], vcc, exec -; SDAG-NEXT: ; %bb.6: ; %Flow3 -; SDAG-NEXT: s_andn2_saveexec_b64 s[12:13], s[12:13] -; SDAG-NEXT: ; %bb.7: ; %LeafBlock -; SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 25, v8 -; SDAG-NEXT: s_andn2_b64 s[4:5], s[4:5], exec -; SDAG-NEXT: s_and_b64 s[14:15], vcc, exec -; SDAG-NEXT: s_mov_b64 s[10:11], exec -; SDAG-NEXT: s_or_b64 s[4:5], s[4:5], s[14:15] -; SDAG-NEXT: ; %bb.8: ; %Flow4 -; SDAG-NEXT: s_or_b64 exec, exec, s[12:13] -; SDAG-NEXT: v_mov_b32_e32 v7, v1 -; SDAG-NEXT: v_mov_b32_e32 v6, v0 -; SDAG-NEXT: s_and_saveexec_b64 s[12:13], s[4:5] -; SDAG-NEXT: s_xor_b64 s[12:13], exec, s[12:13] -; SDAG-NEXT: s_cbranch_execz .LBB4_10 -; SDAG-NEXT: ; %bb.9: ; %itofp-sw-default -; SDAG-NEXT: v_sub_u32_e32 v12, 0x66, v9 +; SDAG-NEXT: s_cbranch_execz .LBB4_7 +; SDAG-NEXT: ; %bb.6: ; %itofp-sw-default +; SDAG-NEXT: v_sub_u32_e32 v12, 0x66, v7 ; SDAG-NEXT: v_sub_u32_e32 v10, 64, v12 -; SDAG-NEXT: v_lshrrev_b64 v[6:7], v12, v[0:1] +; SDAG-NEXT: v_lshrrev_b64 v[8:9], v12, v[0:1] ; SDAG-NEXT: v_lshlrev_b64 v[10:11], v10, v[4:5] -; SDAG-NEXT: v_sub_u32_e32 v13, 38, v9 -; SDAG-NEXT: v_or_b32_e32 v11, v7, v11 -; SDAG-NEXT: v_or_b32_e32 v10, v6, v10 -; SDAG-NEXT: v_lshrrev_b64 v[6:7], v13, v[4:5] +; SDAG-NEXT: v_sub_u32_e32 v13, 38, v7 +; SDAG-NEXT: v_or_b32_e32 v11, v9, v11 +; SDAG-NEXT: v_or_b32_e32 v10, v8, v10 +; SDAG-NEXT: v_lshrrev_b64 v[8:9], v13, v[4:5] ; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v12 -; SDAG-NEXT: v_add_u32_e32 v14, 26, v9 -; SDAG-NEXT: v_cndmask_b32_e32 v7, v7, v11, vcc +; SDAG-NEXT: v_add_u32_e32 v14, 26, v7 +; SDAG-NEXT: v_cndmask_b32_e32 v9, v9, v11, vcc ; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v12 -; SDAG-NEXT: v_cndmask_b32_e32 v6, v6, v10, vcc +; SDAG-NEXT: v_cndmask_b32_e32 v8, v8, v10, vcc ; SDAG-NEXT: v_lshrrev_b64 v[10:11], v13, v[0:1] ; SDAG-NEXT: v_lshlrev_b64 v[12:13], v14, v[4:5] -; SDAG-NEXT: v_subrev_u32_e32 v9, 38, v9 -; SDAG-NEXT: v_cndmask_b32_e64 v15, v6, v0, s[4:5] -; SDAG-NEXT: v_or_b32_e32 v6, v13, v11 -; SDAG-NEXT: v_or_b32_e32 v11, v12, v10 -; SDAG-NEXT: v_lshlrev_b64 v[9:10], v9, v[0:1] +; SDAG-NEXT: v_subrev_u32_e32 v7, 38, v7 +; SDAG-NEXT: v_cndmask_b32_e64 v15, v8, v0, s[4:5] +; SDAG-NEXT: v_lshlrev_b64 v[7:8], v7, v[0:1] +; SDAG-NEXT: v_cndmask_b32_e64 v9, v9, v1, s[4:5] +; SDAG-NEXT: v_or_b32_e32 v11, v13, v11 +; SDAG-NEXT: v_or_b32_e32 v10, v12, v10 ; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v14 -; SDAG-NEXT: v_cndmask_b32_e64 v7, v7, v1, s[4:5] -; SDAG-NEXT: v_cndmask_b32_e32 v6, v10, v6, vcc +; SDAG-NEXT: v_lshlrev_b64 v[0:1], v14, v[0:1] +; SDAG-NEXT: v_cndmask_b32_e32 v8, v8, v11, vcc ; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v14 -; SDAG-NEXT: v_cndmask_b32_e64 v10, v6, v5, s[4:5] -; SDAG-NEXT: v_lshlrev_b64 v[5:6], v14, v[0:1] -; SDAG-NEXT: v_cndmask_b32_e32 v9, v9, v11, vcc -; SDAG-NEXT: v_cndmask_b32_e64 v4, v9, v4, s[4:5] -; SDAG-NEXT: v_cndmask_b32_e32 v6, 0, v6, vcc -; SDAG-NEXT: v_cndmask_b32_e32 v9, 0, v5, vcc -; SDAG-NEXT: v_or_b32_e32 v5, v6, v10 -; SDAG-NEXT: v_or_b32_e32 v4, v9, v4 -; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] -; SDAG-NEXT: s_andn2_b64 s[10:11], s[10:11], exec -; SDAG-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc -; SDAG-NEXT: v_or_b32_e32 v6, v15, v4 -; SDAG-NEXT: .LBB4_10: ; %Flow5 +; SDAG-NEXT: v_cndmask_b32_e32 v7, v7, v10, vcc +; SDAG-NEXT: v_cndmask_b32_e64 v5, v8, v5, s[4:5] +; SDAG-NEXT: v_cndmask_b32_e64 v4, v7, v4, s[4:5] +; SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc +; SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc +; SDAG-NEXT: v_or_b32_e32 v1, v1, v5 +; SDAG-NEXT: v_or_b32_e32 v0, v0, v4 +; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] +; SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc +; SDAG-NEXT: v_or_b32_e32 v8, v15, v0 +; SDAG-NEXT: v_mov_b32_e32 v0, v8 +; SDAG-NEXT: v_mov_b32_e32 v1, v9 +; SDAG-NEXT: .LBB4_7: ; %Flow1 ; SDAG-NEXT: s_or_b64 exec, exec, s[12:13] -; SDAG-NEXT: s_and_saveexec_b64 s[4:5], s[10:11] -; SDAG-NEXT: ; %bb.11: ; %itofp-sw-bb -; SDAG-NEXT: v_lshlrev_b64 v[6:7], 1, v[0:1] -; SDAG-NEXT: ; %bb.12: ; %itofp-sw-epilog +; SDAG-NEXT: .LBB4_8: ; %Flow2 +; SDAG-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11] +; SDAG-NEXT: ; %bb.9: ; %itofp-sw-bb +; SDAG-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1] +; SDAG-NEXT: ; %bb.10: ; %itofp-sw-epilog ; SDAG-NEXT: s_or_b64 exec, exec, s[4:5] -; SDAG-NEXT: v_lshrrev_b32_e32 v0, 2, v6 -; SDAG-NEXT: v_and_or_b32 v0, v0, 1, v6 +; SDAG-NEXT: v_lshrrev_b32_e32 v4, 2, v0 +; SDAG-NEXT: v_and_or_b32 v0, v4, 1, v0 ; SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 1, v0 -; SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v7, vcc +; SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc ; SDAG-NEXT: v_and_b32_e32 v4, 0x4000000, v0 ; SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4 -; SDAG-NEXT: v_alignbit_b32 v6, v1, v0, 2 +; SDAG-NEXT: v_alignbit_b32 v8, v1, v0, 2 ; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc -; SDAG-NEXT: ; %bb.13: ; %itofp-if-then20 -; SDAG-NEXT: v_alignbit_b32 v6, v1, v0, 3 -; SDAG-NEXT: v_mov_b32_e32 v2, v8 -; SDAG-NEXT: ; %bb.14: ; %Flow +; SDAG-NEXT: ; %bb.11: ; %itofp-if-then20 +; SDAG-NEXT: v_alignbit_b32 v8, v1, v0, 3 +; SDAG-NEXT: v_mov_b32_e32 v2, v6 +; SDAG-NEXT: ; %bb.12: ; %Flow ; SDAG-NEXT: s_or_b64 exec, exec, s[4:5] -; SDAG-NEXT: .LBB4_15: ; %Flow7 +; SDAG-NEXT: .LBB4_13: ; %Flow4 ; SDAG-NEXT: s_or_b64 exec, exec, s[8:9] ; SDAG-NEXT: v_and_b32_e32 v0, 0x80000000, v3 ; SDAG-NEXT: v_lshl_add_u32 v1, v2, 23, 1.0 -; SDAG-NEXT: v_and_b32_e32 v2, 0x7fffff, v6 +; SDAG-NEXT: v_and_b32_e32 v2, 0x7fffff, v8 ; SDAG-NEXT: v_or3_b32 v0, v2, v0, v1 ; SDAG-NEXT: v_cvt_f16_f32_e32 v4, v0 -; SDAG-NEXT: .LBB4_16: ; %Flow8 +; SDAG-NEXT: .LBB4_14: ; %Flow5 ; SDAG-NEXT: s_or_b64 exec, exec, s[6:7] ; SDAG-NEXT: v_mov_b32_e32 v0, v4 ; SDAG-NEXT: s_setpc_b64 s[30:31] @@ -1362,145 +1230,127 @@ define half @sitofp_i128_to_f16(i128 %x) { ; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; GISEL-NEXT: v_mov_b32_e32 v4, s4 ; GISEL-NEXT: s_and_saveexec_b64 s[6:7], vcc -; GISEL-NEXT: s_cbranch_execz .LBB4_16 +; GISEL-NEXT: s_cbranch_execz .LBB4_14 ; GISEL-NEXT: ; %bb.1: ; %itofp-if-end -; GISEL-NEXT: v_ashrrev_i32_e32 v8, 31, v3 -; GISEL-NEXT: v_xor_b32_e32 v0, v8, v0 -; GISEL-NEXT: v_xor_b32_e32 v1, v8, v1 -; GISEL-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v8 -; GISEL-NEXT: v_xor_b32_e32 v2, v8, v2 -; GISEL-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v8, vcc -; GISEL-NEXT: v_xor_b32_e32 v3, v8, v3 -; GISEL-NEXT: v_subb_co_u32_e32 v6, vcc, v2, v8, vcc -; GISEL-NEXT: v_subb_co_u32_e32 v7, vcc, v3, v8, vcc -; GISEL-NEXT: v_ffbh_u32_e32 v3, v0 -; GISEL-NEXT: v_ffbh_u32_e32 v2, v1 -; GISEL-NEXT: v_add_u32_e32 v3, 32, v3 -; GISEL-NEXT: v_ffbh_u32_e32 v4, v6 -; GISEL-NEXT: v_min_u32_e32 v2, v2, v3 -; GISEL-NEXT: v_ffbh_u32_e32 v3, v7 -; GISEL-NEXT: v_add_u32_e32 v4, 32, v4 -; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[6:7] -; GISEL-NEXT: v_add_u32_e32 v2, 64, v2 -; GISEL-NEXT: v_min_u32_e32 v3, v3, v4 -; GISEL-NEXT: v_cndmask_b32_e32 v11, v3, v2, vcc -; GISEL-NEXT: v_sub_u32_e32 v9, 0x7f, v11 -; GISEL-NEXT: v_cmp_ge_i32_e32 vcc, 24, v9 -; GISEL-NEXT: ; implicit-def: $vgpr2 +; GISEL-NEXT: v_ashrrev_i32_e32 v6, 31, v3 +; GISEL-NEXT: v_xor_b32_e32 v0, v6, v0 +; GISEL-NEXT: v_xor_b32_e32 v1, v6, v1 +; GISEL-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v6 +; GISEL-NEXT: v_xor_b32_e32 v2, v6, v2 +; GISEL-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v6, vcc +; GISEL-NEXT: v_xor_b32_e32 v3, v6, v3 +; GISEL-NEXT: v_subb_co_u32_e32 v2, vcc, v2, v6, vcc +; GISEL-NEXT: v_ffbh_u32_e32 v5, v0 +; GISEL-NEXT: v_subb_co_u32_e32 v3, vcc, v3, v6, vcc +; GISEL-NEXT: v_ffbh_u32_e32 v4, v1 +; GISEL-NEXT: v_add_u32_e32 v5, 32, v5 +; GISEL-NEXT: v_ffbh_u32_e32 v7, v2 +; GISEL-NEXT: v_min_u32_e32 v4, v4, v5 +; GISEL-NEXT: v_ffbh_u32_e32 v5, v3 +; GISEL-NEXT: v_add_u32_e32 v7, 32, v7 +; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3] +; GISEL-NEXT: v_add_u32_e32 v4, 64, v4 +; GISEL-NEXT: v_min_u32_e32 v5, v5, v7 +; GISEL-NEXT: v_cndmask_b32_e32 v5, v5, v4, vcc +; GISEL-NEXT: v_sub_u32_e32 v8, 0x80, v5 +; GISEL-NEXT: v_sub_u32_e32 v7, 0x7f, v5 +; GISEL-NEXT: v_cmp_ge_i32_e32 vcc, 24, v8 +; GISEL-NEXT: ; implicit-def: $vgpr4 ; GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GISEL-NEXT: ; %bb.2: ; %itofp-if-else -; GISEL-NEXT: v_add_u32_e32 v2, 0xffffff98, v11 +; GISEL-NEXT: v_add_u32_e32 v2, 0xffffff98, v5 ; GISEL-NEXT: v_lshlrev_b64 v[0:1], v2, v[0:1] ; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v2 -; GISEL-NEXT: v_cndmask_b32_e32 v2, 0, v0, vcc +; GISEL-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc +; GISEL-NEXT: ; implicit-def: $vgpr8 ; GISEL-NEXT: ; implicit-def: $vgpr0 -; GISEL-NEXT: ; implicit-def: $vgpr11 -; GISEL-NEXT: ; implicit-def: $vgpr6 -; GISEL-NEXT: ; %bb.3: ; %Flow6 +; GISEL-NEXT: ; implicit-def: $vgpr5 +; GISEL-NEXT: ; implicit-def: $vgpr2 +; GISEL-NEXT: ; %bb.3: ; %Flow3 ; GISEL-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] -; GISEL-NEXT: s_cbranch_execz .LBB4_15 +; GISEL-NEXT: s_cbranch_execz .LBB4_13 ; GISEL-NEXT: ; %bb.4: ; %NodeBlock -; GISEL-NEXT: v_sub_u32_e32 v10, 0x80, v11 -; GISEL-NEXT: v_cmp_le_i32_e32 vcc, 26, v10 -; GISEL-NEXT: s_mov_b64 s[10:11], 0 -; GISEL-NEXT: s_mov_b64 s[4:5], 0 +; GISEL-NEXT: v_cmp_le_i32_e32 vcc, 26, v8 +; GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GISEL-NEXT: s_xor_b64 s[10:11], exec, s[4:5] +; GISEL-NEXT: s_cbranch_execz .LBB4_8 +; GISEL-NEXT: ; %bb.5: ; %LeafBlock +; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 26, v8 ; GISEL-NEXT: s_and_saveexec_b64 s[12:13], vcc -; GISEL-NEXT: s_xor_b64 s[12:13], exec, s[12:13] -; GISEL-NEXT: ; %bb.5: ; %LeafBlock1 -; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 26, v10 -; GISEL-NEXT: s_andn2_b64 s[4:5], 0, exec -; GISEL-NEXT: s_and_b64 s[14:15], exec, vcc -; GISEL-NEXT: s_or_b64 s[4:5], s[4:5], s[14:15] -; GISEL-NEXT: ; %bb.6: ; %Flow3 -; GISEL-NEXT: s_andn2_saveexec_b64 s[12:13], s[12:13] -; GISEL-NEXT: ; %bb.7: ; %LeafBlock -; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 25, v10 -; GISEL-NEXT: s_andn2_b64 s[10:11], 0, exec -; GISEL-NEXT: s_and_b64 s[14:15], exec, -1 -; GISEL-NEXT: s_or_b64 s[10:11], s[10:11], s[14:15] -; GISEL-NEXT: s_andn2_b64 s[4:5], s[4:5], exec -; GISEL-NEXT: s_and_b64 s[14:15], exec, vcc -; GISEL-NEXT: s_or_b64 s[4:5], s[4:5], s[14:15] -; GISEL-NEXT: ; %bb.8: ; %Flow4 -; GISEL-NEXT: s_or_b64 exec, exec, s[12:13] -; GISEL-NEXT: v_mov_b32_e32 v5, v3 -; GISEL-NEXT: v_mov_b32_e32 v4, v2 -; GISEL-NEXT: v_mov_b32_e32 v3, v1 -; GISEL-NEXT: v_mov_b32_e32 v2, v0 -; GISEL-NEXT: s_and_saveexec_b64 s[12:13], s[4:5] -; GISEL-NEXT: s_xor_b64 s[12:13], exec, s[12:13] -; GISEL-NEXT: s_cbranch_execz .LBB4_10 -; GISEL-NEXT: ; %bb.9: ; %itofp-sw-default -; GISEL-NEXT: v_sub_u32_e32 v12, 0x66, v11 -; GISEL-NEXT: v_sub_u32_e32 v4, 64, v12 -; GISEL-NEXT: v_lshrrev_b64 v[2:3], v12, v[0:1] -; GISEL-NEXT: v_lshlrev_b64 v[4:5], v4, v[6:7] -; GISEL-NEXT: v_subrev_u32_e32 v13, 64, v12 -; GISEL-NEXT: v_or_b32_e32 v4, v2, v4 -; GISEL-NEXT: v_or_b32_e32 v5, v3, v5 -; GISEL-NEXT: v_lshrrev_b64 v[2:3], v13, v[6:7] -; GISEL-NEXT: v_add_u32_e32 v13, 26, v11 -; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v12 -; GISEL-NEXT: v_sub_u32_e32 v11, 64, v13 -; GISEL-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc -; GISEL-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc -; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v12 -; GISEL-NEXT: v_lshrrev_b64 v[4:5], v13, -1 +; GISEL-NEXT: s_cbranch_execz .LBB4_7 +; GISEL-NEXT: ; %bb.6: ; %itofp-sw-default +; GISEL-NEXT: v_sub_u32_e32 v4, 0x66, v5 +; GISEL-NEXT: v_sub_u32_e32 v11, 64, v4 +; GISEL-NEXT: v_lshrrev_b64 v[9:10], v4, v[0:1] +; GISEL-NEXT: v_lshlrev_b64 v[11:12], v11, v[2:3] +; GISEL-NEXT: v_subrev_u32_e32 v13, 64, v4 +; GISEL-NEXT: v_or_b32_e32 v11, v9, v11 +; GISEL-NEXT: v_or_b32_e32 v12, v10, v12 +; GISEL-NEXT: v_lshrrev_b64 v[9:10], v13, v[2:3] +; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v4 +; GISEL-NEXT: v_add_u32_e32 v5, 26, v5 +; GISEL-NEXT: v_cndmask_b32_e32 v9, v9, v11, vcc +; GISEL-NEXT: v_cndmask_b32_e32 v10, v10, v12, vcc +; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4 +; GISEL-NEXT: v_sub_u32_e32 v11, 64, v5 +; GISEL-NEXT: v_cndmask_b32_e32 v13, v9, v0, vcc +; GISEL-NEXT: v_cndmask_b32_e32 v4, v10, v1, vcc +; GISEL-NEXT: v_lshrrev_b64 v[9:10], v5, -1 ; GISEL-NEXT: v_lshlrev_b64 v[11:12], v11, -1 -; GISEL-NEXT: v_subrev_u32_e32 v14, 64, v13 -; GISEL-NEXT: v_or_b32_e32 v15, v4, v11 -; GISEL-NEXT: v_or_b32_e32 v16, v5, v12 +; GISEL-NEXT: v_subrev_u32_e32 v14, 64, v5 +; GISEL-NEXT: v_or_b32_e32 v15, v9, v11 +; GISEL-NEXT: v_or_b32_e32 v16, v10, v12 ; GISEL-NEXT: v_lshrrev_b64 v[11:12], v14, -1 -; GISEL-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc -; GISEL-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc -; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v13 +; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v5 ; GISEL-NEXT: v_cndmask_b32_e32 v11, v11, v15, vcc ; GISEL-NEXT: v_cndmask_b32_e32 v12, v12, v16, vcc -; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v13 -; GISEL-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc -; GISEL-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc -; GISEL-NEXT: v_cndmask_b32_e64 v11, v11, -1, s[4:5] -; GISEL-NEXT: v_cndmask_b32_e64 v12, v12, -1, s[4:5] -; GISEL-NEXT: v_and_b32_e32 v4, v4, v6 -; GISEL-NEXT: v_and_b32_e32 v5, v5, v7 -; GISEL-NEXT: v_and_or_b32 v4, v11, v0, v4 -; GISEL-NEXT: v_and_or_b32 v5, v12, v1, v5 -; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] -; GISEL-NEXT: s_andn2_b64 s[4:5], s[10:11], exec -; GISEL-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc -; GISEL-NEXT: s_and_b64 s[10:11], exec, 0 -; GISEL-NEXT: v_or_b32_e32 v2, v2, v4 -; GISEL-NEXT: s_or_b64 s[10:11], s[4:5], s[10:11] -; GISEL-NEXT: .LBB4_10: ; %Flow5 +; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v5 +; GISEL-NEXT: v_cndmask_b32_e32 v9, 0, v9, vcc +; GISEL-NEXT: v_cndmask_b32_e32 v10, 0, v10, vcc +; GISEL-NEXT: v_cndmask_b32_e64 v5, v11, -1, s[4:5] +; GISEL-NEXT: v_cndmask_b32_e64 v11, v12, -1, s[4:5] +; GISEL-NEXT: v_and_b32_e32 v2, v9, v2 +; GISEL-NEXT: v_and_b32_e32 v3, v10, v3 +; GISEL-NEXT: v_and_or_b32 v0, v5, v0, v2 +; GISEL-NEXT: v_and_or_b32 v1, v11, v1, v3 +; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] +; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc +; GISEL-NEXT: v_or_b32_e32 v3, v13, v0 +; GISEL-NEXT: v_mov_b32_e32 v0, v3 +; GISEL-NEXT: v_mov_b32_e32 v1, v4 +; GISEL-NEXT: v_mov_b32_e32 v2, v5 +; GISEL-NEXT: v_mov_b32_e32 v3, v6 +; GISEL-NEXT: .LBB4_7: ; %Flow1 ; GISEL-NEXT: s_or_b64 exec, exec, s[12:13] -; GISEL-NEXT: s_and_saveexec_b64 s[4:5], s[10:11] -; GISEL-NEXT: ; %bb.11: ; %itofp-sw-bb -; GISEL-NEXT: v_lshlrev_b64 v[2:3], 1, v[0:1] -; GISEL-NEXT: ; %bb.12: ; %itofp-sw-epilog +; GISEL-NEXT: .LBB4_8: ; %Flow2 +; GISEL-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11] +; GISEL-NEXT: ; %bb.9: ; %itofp-sw-bb +; GISEL-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1] +; GISEL-NEXT: ; %bb.10: ; %itofp-sw-epilog ; GISEL-NEXT: s_or_b64 exec, exec, s[4:5] -; GISEL-NEXT: v_bfe_u32 v0, v2, 2, 1 -; GISEL-NEXT: v_or_b32_e32 v0, v2, v0 +; GISEL-NEXT: v_bfe_u32 v2, v0, 2, 1 +; GISEL-NEXT: v_or_b32_e32 v0, v0, v2 ; GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 1, v0 -; GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc -; GISEL-NEXT: v_lshrrev_b64 v[2:3], 2, v[0:1] -; GISEL-NEXT: v_and_b32_e32 v3, 0x4000000, v0 -; GISEL-NEXT: v_mov_b32_e32 v4, 0 -; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[3:4] +; GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc +; GISEL-NEXT: v_and_b32_e32 v2, 0x4000000, v0 +; GISEL-NEXT: v_mov_b32_e32 v3, 0 +; GISEL-NEXT: v_lshrrev_b64 v[4:5], 2, v[0:1] +; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3] ; GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc -; GISEL-NEXT: ; %bb.13: ; %itofp-if-then20 -; GISEL-NEXT: v_lshrrev_b64 v[2:3], 3, v[0:1] -; GISEL-NEXT: v_mov_b32_e32 v9, v10 -; GISEL-NEXT: ; %bb.14: ; %Flow +; GISEL-NEXT: ; %bb.11: ; %itofp-if-then20 +; GISEL-NEXT: v_lshrrev_b64 v[4:5], 3, v[0:1] +; GISEL-NEXT: v_mov_b32_e32 v7, v8 +; GISEL-NEXT: ; %bb.12: ; %Flow ; GISEL-NEXT: s_or_b64 exec, exec, s[4:5] -; GISEL-NEXT: .LBB4_15: ; %Flow7 +; GISEL-NEXT: .LBB4_13: ; %Flow4 ; GISEL-NEXT: s_or_b64 exec, exec, s[8:9] -; GISEL-NEXT: v_and_b32_e32 v0, 0x80000000, v8 -; GISEL-NEXT: v_lshl_add_u32 v1, v9, 23, 1.0 -; GISEL-NEXT: v_and_b32_e32 v2, 0x7fffff, v2 +; GISEL-NEXT: v_and_b32_e32 v0, 0x80000000, v6 +; GISEL-NEXT: v_lshl_add_u32 v1, v7, 23, 1.0 +; GISEL-NEXT: v_and_b32_e32 v2, 0x7fffff, v4 ; GISEL-NEXT: v_or3_b32 v0, v2, v0, v1 ; GISEL-NEXT: v_cvt_f16_f32_e32 v4, v0 -; GISEL-NEXT: .LBB4_16: ; %Flow8 +; GISEL-NEXT: .LBB4_14: ; %Flow5 ; GISEL-NEXT: s_or_b64 exec, exec, s[6:7] ; GISEL-NEXT: v_mov_b32_e32 v0, v4 ; GISEL-NEXT: s_setpc_b64 s[30:31] @@ -1517,7 +1367,7 @@ define half @uitofp_i128_to_f16(i128 %x) { ; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] ; SDAG-NEXT: v_mov_b32_e32 v4, 0 ; SDAG-NEXT: s_and_saveexec_b64 s[6:7], vcc -; SDAG-NEXT: s_cbranch_execz .LBB5_16 +; SDAG-NEXT: s_cbranch_execz .LBB5_14 ; SDAG-NEXT: ; %bb.1: ; %itofp-if-end ; SDAG-NEXT: v_ffbh_u32_e32 v4, v2 ; SDAG-NEXT: v_add_u32_e32 v4, 32, v4 @@ -1529,114 +1379,100 @@ define half @uitofp_i128_to_f16(i128 %x) { ; SDAG-NEXT: v_min_u32_e32 v5, v5, v6 ; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3] ; SDAG-NEXT: v_add_u32_e32 v5, 64, v5 -; SDAG-NEXT: v_cndmask_b32_e32 v8, v5, v4, vcc -; SDAG-NEXT: v_sub_u32_e32 v6, 0x7f, v8 -; SDAG-NEXT: v_mov_b32_e32 v5, v1 -; SDAG-NEXT: v_cmp_gt_i32_e32 vcc, 25, v6 -; SDAG-NEXT: v_mov_b32_e32 v4, v0 -; SDAG-NEXT: ; implicit-def: $vgpr9 +; SDAG-NEXT: v_cndmask_b32_e32 v6, v5, v4, vcc +; SDAG-NEXT: v_sub_u32_e32 v5, 0x80, v6 +; SDAG-NEXT: v_sub_u32_e32 v4, 0x7f, v6 +; SDAG-NEXT: v_cmp_gt_i32_e32 vcc, 25, v5 +; SDAG-NEXT: ; implicit-def: $vgpr7 ; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc ; SDAG-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; SDAG-NEXT: ; %bb.2: ; %itofp-if-else -; SDAG-NEXT: v_add_u32_e32 v2, 0xffffff98, v8 +; SDAG-NEXT: v_add_u32_e32 v2, 0xffffff98, v6 ; SDAG-NEXT: v_lshlrev_b64 v[0:1], v2, v[0:1] ; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v2 -; SDAG-NEXT: v_cndmask_b32_e32 v9, 0, v0, vcc +; SDAG-NEXT: v_cndmask_b32_e32 v7, 0, v0, vcc +; SDAG-NEXT: ; implicit-def: $vgpr5 ; SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 -; SDAG-NEXT: ; implicit-def: $vgpr8 +; SDAG-NEXT: ; implicit-def: $vgpr6 ; SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3 -; SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5 -; SDAG-NEXT: ; %bb.3: ; %Flow6 +; SDAG-NEXT: ; %bb.3: ; %Flow3 ; SDAG-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] -; SDAG-NEXT: s_cbranch_execz .LBB5_15 +; SDAG-NEXT: s_cbranch_execz .LBB5_13 ; SDAG-NEXT: ; %bb.4: ; %NodeBlock -; SDAG-NEXT: v_sub_u32_e32 v7, 0x80, v8 -; SDAG-NEXT: v_cmp_lt_i32_e32 vcc, 25, v7 -; SDAG-NEXT: s_mov_b64 s[10:11], 0 -; SDAG-NEXT: s_mov_b64 s[4:5], 0 +; SDAG-NEXT: v_cmp_lt_i32_e32 vcc, 25, v5 +; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc +; SDAG-NEXT: s_xor_b64 s[10:11], exec, s[4:5] +; SDAG-NEXT: s_cbranch_execz .LBB5_8 +; SDAG-NEXT: ; %bb.5: ; %LeafBlock +; SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 26, v5 ; SDAG-NEXT: s_and_saveexec_b64 s[12:13], vcc -; SDAG-NEXT: s_xor_b64 s[12:13], exec, s[12:13] -; SDAG-NEXT: ; %bb.5: ; %LeafBlock1 -; SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 26, v7 -; SDAG-NEXT: s_and_b64 s[4:5], vcc, exec -; SDAG-NEXT: ; %bb.6: ; %Flow3 -; SDAG-NEXT: s_andn2_saveexec_b64 s[12:13], s[12:13] -; SDAG-NEXT: ; %bb.7: ; %LeafBlock -; SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 25, v7 -; SDAG-NEXT: s_andn2_b64 s[4:5], s[4:5], exec -; SDAG-NEXT: s_and_b64 s[14:15], vcc, exec -; SDAG-NEXT: s_mov_b64 s[10:11], exec -; SDAG-NEXT: s_or_b64 s[4:5], s[4:5], s[14:15] -; SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5 -; SDAG-NEXT: ; %bb.8: ; %Flow4 -; SDAG-NEXT: s_or_b64 exec, exec, s[12:13] -; SDAG-NEXT: s_and_saveexec_b64 s[12:13], s[4:5] -; SDAG-NEXT: s_xor_b64 s[12:13], exec, s[12:13] -; SDAG-NEXT: s_cbranch_execz .LBB5_10 -; SDAG-NEXT: ; %bb.9: ; %itofp-sw-default -; SDAG-NEXT: v_sub_u32_e32 v11, 0x66, v8 +; SDAG-NEXT: s_cbranch_execz .LBB5_7 +; SDAG-NEXT: ; %bb.6: ; %itofp-sw-default +; SDAG-NEXT: v_sub_u32_e32 v11, 0x66, v6 ; SDAG-NEXT: v_sub_u32_e32 v9, 64, v11 -; SDAG-NEXT: v_lshrrev_b64 v[4:5], v11, v[0:1] +; SDAG-NEXT: v_lshrrev_b64 v[7:8], v11, v[0:1] ; SDAG-NEXT: v_lshlrev_b64 v[9:10], v9, v[2:3] -; SDAG-NEXT: v_sub_u32_e32 v12, 38, v8 -; SDAG-NEXT: v_or_b32_e32 v10, v5, v10 -; SDAG-NEXT: v_or_b32_e32 v9, v4, v9 -; SDAG-NEXT: v_lshrrev_b64 v[4:5], v12, v[2:3] +; SDAG-NEXT: v_sub_u32_e32 v12, 38, v6 +; SDAG-NEXT: v_or_b32_e32 v10, v8, v10 +; SDAG-NEXT: v_or_b32_e32 v9, v7, v9 +; SDAG-NEXT: v_lshrrev_b64 v[7:8], v12, v[2:3] ; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v11 -; SDAG-NEXT: v_add_u32_e32 v13, 26, v8 -; SDAG-NEXT: v_cndmask_b32_e32 v5, v5, v10, vcc +; SDAG-NEXT: v_add_u32_e32 v13, 26, v6 +; SDAG-NEXT: v_cndmask_b32_e32 v8, v8, v10, vcc ; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v11 -; SDAG-NEXT: v_cndmask_b32_e32 v4, v4, v9, vcc +; SDAG-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc ; SDAG-NEXT: v_lshrrev_b64 v[9:10], v12, v[0:1] ; SDAG-NEXT: v_lshlrev_b64 v[11:12], v13, v[2:3] -; SDAG-NEXT: v_subrev_u32_e32 v8, 38, v8 -; SDAG-NEXT: v_cndmask_b32_e64 v14, v4, v0, s[4:5] -; SDAG-NEXT: v_or_b32_e32 v4, v12, v10 -; SDAG-NEXT: v_or_b32_e32 v10, v11, v9 -; SDAG-NEXT: v_lshlrev_b64 v[8:9], v8, v[0:1] +; SDAG-NEXT: v_subrev_u32_e32 v6, 38, v6 +; SDAG-NEXT: v_cndmask_b32_e64 v14, v7, v0, s[4:5] +; SDAG-NEXT: v_lshlrev_b64 v[6:7], v6, v[0:1] +; SDAG-NEXT: v_cndmask_b32_e64 v8, v8, v1, s[4:5] +; SDAG-NEXT: v_or_b32_e32 v10, v12, v10 +; SDAG-NEXT: v_or_b32_e32 v9, v11, v9 ; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v13 -; SDAG-NEXT: v_cndmask_b32_e64 v5, v5, v1, s[4:5] -; SDAG-NEXT: v_cndmask_b32_e32 v4, v9, v4, vcc +; SDAG-NEXT: v_lshlrev_b64 v[0:1], v13, v[0:1] +; SDAG-NEXT: v_cndmask_b32_e32 v7, v7, v10, vcc ; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v13 -; SDAG-NEXT: v_cndmask_b32_e64 v9, v4, v3, s[4:5] -; SDAG-NEXT: v_lshlrev_b64 v[3:4], v13, v[0:1] -; SDAG-NEXT: v_cndmask_b32_e32 v8, v8, v10, vcc -; SDAG-NEXT: v_cndmask_b32_e64 v2, v8, v2, s[4:5] -; SDAG-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc -; SDAG-NEXT: v_cndmask_b32_e32 v8, 0, v3, vcc -; SDAG-NEXT: v_or_b32_e32 v3, v4, v9 -; SDAG-NEXT: v_or_b32_e32 v2, v8, v2 -; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3] -; SDAG-NEXT: s_andn2_b64 s[10:11], s[10:11], exec -; SDAG-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc -; SDAG-NEXT: v_or_b32_e32 v4, v14, v2 -; SDAG-NEXT: .LBB5_10: ; %Flow5 +; SDAG-NEXT: v_cndmask_b32_e32 v6, v6, v9, vcc +; SDAG-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[4:5] +; SDAG-NEXT: v_cndmask_b32_e64 v2, v6, v2, s[4:5] +; SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc +; SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc +; SDAG-NEXT: v_or_b32_e32 v1, v1, v3 +; SDAG-NEXT: v_or_b32_e32 v0, v0, v2 +; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] +; SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc +; SDAG-NEXT: v_or_b32_e32 v7, v14, v0 +; SDAG-NEXT: v_mov_b32_e32 v0, v7 +; SDAG-NEXT: v_mov_b32_e32 v1, v8 +; SDAG-NEXT: .LBB5_7: ; %Flow1 ; SDAG-NEXT: s_or_b64 exec, exec, s[12:13] -; SDAG-NEXT: s_and_saveexec_b64 s[4:5], s[10:11] -; SDAG-NEXT: ; %bb.11: ; %itofp-sw-bb -; SDAG-NEXT: v_lshlrev_b64 v[4:5], 1, v[0:1] -; SDAG-NEXT: ; %bb.12: ; %itofp-sw-epilog +; SDAG-NEXT: .LBB5_8: ; %Flow2 +; SDAG-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11] +; SDAG-NEXT: ; %bb.9: ; %itofp-sw-bb +; SDAG-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1] +; SDAG-NEXT: ; %bb.10: ; %itofp-sw-epilog ; SDAG-NEXT: s_or_b64 exec, exec, s[4:5] -; SDAG-NEXT: v_lshrrev_b32_e32 v0, 2, v4 -; SDAG-NEXT: v_and_or_b32 v0, v0, 1, v4 +; SDAG-NEXT: v_lshrrev_b32_e32 v2, 2, v0 +; SDAG-NEXT: v_and_or_b32 v0, v2, 1, v0 ; SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 1, v0 -; SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v5, vcc +; SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc ; SDAG-NEXT: v_and_b32_e32 v2, 0x4000000, v0 ; SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2 -; SDAG-NEXT: v_alignbit_b32 v9, v1, v0, 2 +; SDAG-NEXT: v_alignbit_b32 v7, v1, v0, 2 ; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc -; SDAG-NEXT: ; %bb.13: ; %itofp-if-then20 -; SDAG-NEXT: v_alignbit_b32 v9, v1, v0, 3 -; SDAG-NEXT: v_mov_b32_e32 v6, v7 -; SDAG-NEXT: ; %bb.14: ; %Flow +; SDAG-NEXT: ; %bb.11: ; %itofp-if-then20 +; SDAG-NEXT: v_alignbit_b32 v7, v1, v0, 3 +; SDAG-NEXT: v_mov_b32_e32 v4, v5 +; SDAG-NEXT: ; %bb.12: ; %Flow ; SDAG-NEXT: s_or_b64 exec, exec, s[4:5] -; SDAG-NEXT: .LBB5_15: ; %Flow7 +; SDAG-NEXT: .LBB5_13: ; %Flow4 ; SDAG-NEXT: s_or_b64 exec, exec, s[8:9] -; SDAG-NEXT: v_and_b32_e32 v0, 0x7fffff, v9 -; SDAG-NEXT: v_lshl_or_b32 v0, v6, 23, v0 +; SDAG-NEXT: v_and_b32_e32 v0, 0x7fffff, v7 +; SDAG-NEXT: v_lshl_or_b32 v0, v4, 23, v0 ; SDAG-NEXT: v_add_u32_e32 v0, 1.0, v0 ; SDAG-NEXT: v_cvt_f16_f32_e32 v4, v0 -; SDAG-NEXT: .LBB5_16: ; %Flow8 +; SDAG-NEXT: .LBB5_14: ; %Flow5 ; SDAG-NEXT: s_or_b64 exec, exec, s[6:7] ; SDAG-NEXT: v_mov_b32_e32 v0, v4 ; SDAG-NEXT: s_setpc_b64 s[30:31] @@ -1644,145 +1480,125 @@ define half @uitofp_i128_to_f16(i128 %x) { ; GISEL-LABEL: uitofp_i128_to_f16: ; GISEL: ; %bb.0: ; %itofp-entry ; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GISEL-NEXT: v_mov_b32_e32 v4, v2 -; GISEL-NEXT: v_mov_b32_e32 v5, v3 -; GISEL-NEXT: v_or_b32_e32 v2, v0, v4 -; GISEL-NEXT: v_or_b32_e32 v3, v1, v5 +; GISEL-NEXT: v_or_b32_e32 v4, v0, v2 +; GISEL-NEXT: v_or_b32_e32 v5, v1, v3 ; GISEL-NEXT: s_mov_b32 s4, 0 -; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3] -; GISEL-NEXT: v_mov_b32_e32 v2, s4 +; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] +; GISEL-NEXT: v_mov_b32_e32 v4, s4 ; GISEL-NEXT: s_and_saveexec_b64 s[6:7], vcc -; GISEL-NEXT: s_cbranch_execz .LBB5_16 +; GISEL-NEXT: s_cbranch_execz .LBB5_14 ; GISEL-NEXT: ; %bb.1: ; %itofp-if-end -; GISEL-NEXT: v_ffbh_u32_e32 v3, v0 -; GISEL-NEXT: v_ffbh_u32_e32 v2, v1 -; GISEL-NEXT: v_add_u32_e32 v3, 32, v3 -; GISEL-NEXT: v_ffbh_u32_e32 v6, v4 -; GISEL-NEXT: v_min_u32_e32 v2, v2, v3 -; GISEL-NEXT: v_ffbh_u32_e32 v3, v5 +; GISEL-NEXT: v_ffbh_u32_e32 v5, v0 +; GISEL-NEXT: v_ffbh_u32_e32 v4, v1 +; GISEL-NEXT: v_add_u32_e32 v5, 32, v5 +; GISEL-NEXT: v_ffbh_u32_e32 v6, v2 +; GISEL-NEXT: v_min_u32_e32 v4, v4, v5 +; GISEL-NEXT: v_ffbh_u32_e32 v5, v3 ; GISEL-NEXT: v_add_u32_e32 v6, 32, v6 -; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[4:5] -; GISEL-NEXT: v_add_u32_e32 v2, 64, v2 -; GISEL-NEXT: v_min_u32_e32 v3, v3, v6 -; GISEL-NEXT: v_cndmask_b32_e32 v12, v3, v2, vcc -; GISEL-NEXT: v_sub_u32_e32 v10, 0x7f, v12 -; GISEL-NEXT: v_cmp_ge_i32_e32 vcc, 24, v10 -; GISEL-NEXT: ; implicit-def: $vgpr2 +; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3] +; GISEL-NEXT: v_add_u32_e32 v4, 64, v4 +; GISEL-NEXT: v_min_u32_e32 v5, v5, v6 +; GISEL-NEXT: v_cndmask_b32_e32 v5, v5, v4, vcc +; GISEL-NEXT: v_sub_u32_e32 v7, 0x80, v5 +; GISEL-NEXT: v_sub_u32_e32 v6, 0x7f, v5 +; GISEL-NEXT: v_cmp_ge_i32_e32 vcc, 24, v7 +; GISEL-NEXT: ; implicit-def: $vgpr4 ; GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5] ; GISEL-NEXT: ; %bb.2: ; %itofp-if-else -; GISEL-NEXT: v_add_u32_e32 v2, 0xffffff98, v12 +; GISEL-NEXT: v_add_u32_e32 v2, 0xffffff98, v5 ; GISEL-NEXT: v_lshlrev_b64 v[0:1], v2, v[0:1] ; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v2 -; GISEL-NEXT: v_cndmask_b32_e32 v2, 0, v0, vcc +; GISEL-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc +; GISEL-NEXT: ; implicit-def: $vgpr7 ; GISEL-NEXT: ; implicit-def: $vgpr0 -; GISEL-NEXT: ; implicit-def: $vgpr12 -; GISEL-NEXT: ; implicit-def: $vgpr4 -; GISEL-NEXT: ; %bb.3: ; %Flow6 +; GISEL-NEXT: ; implicit-def: $vgpr5 +; GISEL-NEXT: ; implicit-def: $vgpr2 +; GISEL-NEXT: ; %bb.3: ; %Flow3 ; GISEL-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] -; GISEL-NEXT: s_cbranch_execz .LBB5_15 +; GISEL-NEXT: s_cbranch_execz .LBB5_13 ; GISEL-NEXT: ; %bb.4: ; %NodeBlock -; GISEL-NEXT: v_sub_u32_e32 v11, 0x80, v12 -; GISEL-NEXT: v_cmp_le_i32_e32 vcc, 26, v11 -; GISEL-NEXT: s_mov_b64 s[10:11], 0 -; GISEL-NEXT: s_mov_b64 s[4:5], 0 +; GISEL-NEXT: v_cmp_le_i32_e32 vcc, 26, v7 +; GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GISEL-NEXT: s_xor_b64 s[10:11], exec, s[4:5] +; GISEL-NEXT: s_cbranch_execz .LBB5_8 +; GISEL-NEXT: ; %bb.5: ; %LeafBlock +; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 26, v7 ; GISEL-NEXT: s_and_saveexec_b64 s[12:13], vcc -; GISEL-NEXT: s_xor_b64 s[12:13], exec, s[12:13] -; GISEL-NEXT: ; %bb.5: ; %LeafBlock1 -; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 26, v11 -; GISEL-NEXT: s_andn2_b64 s[4:5], 0, exec -; GISEL-NEXT: s_and_b64 s[14:15], exec, vcc -; GISEL-NEXT: s_or_b64 s[4:5], s[4:5], s[14:15] -; GISEL-NEXT: ; %bb.6: ; %Flow3 -; GISEL-NEXT: s_andn2_saveexec_b64 s[12:13], s[12:13] -; GISEL-NEXT: ; %bb.7: ; %LeafBlock -; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 25, v11 -; GISEL-NEXT: s_andn2_b64 s[10:11], 0, exec -; GISEL-NEXT: s_and_b64 s[14:15], exec, -1 -; GISEL-NEXT: s_or_b64 s[10:11], s[10:11], s[14:15] -; GISEL-NEXT: s_andn2_b64 s[4:5], s[4:5], exec -; GISEL-NEXT: s_and_b64 s[14:15], exec, vcc -; GISEL-NEXT: s_or_b64 s[4:5], s[4:5], s[14:15] -; GISEL-NEXT: ; %bb.8: ; %Flow4 -; GISEL-NEXT: s_or_b64 exec, exec, s[12:13] -; GISEL-NEXT: v_mov_b32_e32 v9, v3 -; GISEL-NEXT: v_mov_b32_e32 v7, v1 -; GISEL-NEXT: v_mov_b32_e32 v6, v0 -; GISEL-NEXT: v_mov_b32_e32 v8, v2 -; GISEL-NEXT: s_and_saveexec_b64 s[12:13], s[4:5] -; GISEL-NEXT: s_xor_b64 s[12:13], exec, s[12:13] -; GISEL-NEXT: s_cbranch_execz .LBB5_10 -; GISEL-NEXT: ; %bb.9: ; %itofp-sw-default -; GISEL-NEXT: v_sub_u32_e32 v8, 0x66, v12 -; GISEL-NEXT: v_sub_u32_e32 v6, 64, v8 -; GISEL-NEXT: v_lshrrev_b64 v[2:3], v8, v[0:1] -; GISEL-NEXT: v_lshlrev_b64 v[6:7], v6, v[4:5] -; GISEL-NEXT: v_subrev_u32_e32 v9, 64, v8 -; GISEL-NEXT: v_or_b32_e32 v6, v2, v6 -; GISEL-NEXT: v_or_b32_e32 v7, v3, v7 -; GISEL-NEXT: v_lshrrev_b64 v[2:3], v9, v[4:5] -; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v8 -; GISEL-NEXT: v_add_u32_e32 v12, 26, v12 -; GISEL-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc -; GISEL-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc -; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v8 -; GISEL-NEXT: v_sub_u32_e32 v8, 64, v12 -; GISEL-NEXT: v_cndmask_b32_e32 v6, v2, v0, vcc -; GISEL-NEXT: v_cndmask_b32_e32 v7, v3, v1, vcc -; GISEL-NEXT: v_lshrrev_b64 v[2:3], v12, -1 -; GISEL-NEXT: v_lshlrev_b64 v[8:9], v8, -1 -; GISEL-NEXT: v_subrev_u32_e32 v13, 64, v12 -; GISEL-NEXT: v_or_b32_e32 v14, v2, v8 -; GISEL-NEXT: v_or_b32_e32 v15, v3, v9 -; GISEL-NEXT: v_lshrrev_b64 v[8:9], v13, -1 -; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v12 -; GISEL-NEXT: v_cndmask_b32_e32 v8, v8, v14, vcc -; GISEL-NEXT: v_cndmask_b32_e32 v9, v9, v15, vcc -; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v12 -; GISEL-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc -; GISEL-NEXT: v_cndmask_b32_e32 v3, 0, v3, vcc -; GISEL-NEXT: v_cndmask_b32_e64 v8, v8, -1, s[4:5] -; GISEL-NEXT: v_cndmask_b32_e64 v9, v9, -1, s[4:5] -; GISEL-NEXT: v_and_b32_e32 v2, v2, v4 -; GISEL-NEXT: v_and_b32_e32 v3, v3, v5 -; GISEL-NEXT: v_and_or_b32 v2, v8, v0, v2 -; GISEL-NEXT: v_and_or_b32 v3, v9, v1, v3 -; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3] -; GISEL-NEXT: s_andn2_b64 s[4:5], s[10:11], exec -; GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc -; GISEL-NEXT: s_and_b64 s[10:11], exec, 0 -; GISEL-NEXT: v_or_b32_e32 v6, v6, v2 -; GISEL-NEXT: s_or_b64 s[10:11], s[4:5], s[10:11] -; GISEL-NEXT: .LBB5_10: ; %Flow5 +; GISEL-NEXT: s_cbranch_execz .LBB5_7 +; GISEL-NEXT: ; %bb.6: ; %itofp-sw-default +; GISEL-NEXT: v_sub_u32_e32 v4, 0x66, v5 +; GISEL-NEXT: v_sub_u32_e32 v10, 64, v4 +; GISEL-NEXT: v_lshrrev_b64 v[8:9], v4, v[0:1] +; GISEL-NEXT: v_lshlrev_b64 v[10:11], v10, v[2:3] +; GISEL-NEXT: v_subrev_u32_e32 v12, 64, v4 +; GISEL-NEXT: v_or_b32_e32 v10, v8, v10 +; GISEL-NEXT: v_or_b32_e32 v11, v9, v11 +; GISEL-NEXT: v_lshrrev_b64 v[8:9], v12, v[2:3] +; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v4 +; GISEL-NEXT: v_add_u32_e32 v5, 26, v5 +; GISEL-NEXT: v_cndmask_b32_e32 v8, v8, v10, vcc +; GISEL-NEXT: v_cndmask_b32_e32 v9, v9, v11, vcc +; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4 +; GISEL-NEXT: v_sub_u32_e32 v10, 64, v5 +; GISEL-NEXT: v_cndmask_b32_e32 v12, v8, v0, vcc +; GISEL-NEXT: v_cndmask_b32_e32 v4, v9, v1, vcc +; GISEL-NEXT: v_lshrrev_b64 v[8:9], v5, -1 +; GISEL-NEXT: v_lshlrev_b64 v[10:11], v10, -1 +; GISEL-NEXT: v_subrev_u32_e32 v13, 64, v5 +; GISEL-NEXT: v_or_b32_e32 v14, v8, v10 +; GISEL-NEXT: v_or_b32_e32 v15, v9, v11 +; GISEL-NEXT: v_lshrrev_b64 v[10:11], v13, -1 +; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v5 +; GISEL-NEXT: v_cndmask_b32_e32 v10, v10, v14, vcc +; GISEL-NEXT: v_cndmask_b32_e32 v11, v11, v15, vcc +; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v5 +; GISEL-NEXT: v_cndmask_b32_e32 v8, 0, v8, vcc +; GISEL-NEXT: v_cndmask_b32_e32 v9, 0, v9, vcc +; GISEL-NEXT: v_cndmask_b32_e64 v5, v10, -1, s[4:5] +; GISEL-NEXT: v_cndmask_b32_e64 v10, v11, -1, s[4:5] +; GISEL-NEXT: v_and_b32_e32 v2, v8, v2 +; GISEL-NEXT: v_and_b32_e32 v3, v9, v3 +; GISEL-NEXT: v_and_or_b32 v0, v5, v0, v2 +; GISEL-NEXT: v_and_or_b32 v1, v10, v1, v3 +; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] +; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc +; GISEL-NEXT: v_or_b32_e32 v3, v12, v0 +; GISEL-NEXT: v_mov_b32_e32 v0, v3 +; GISEL-NEXT: v_mov_b32_e32 v1, v4 +; GISEL-NEXT: v_mov_b32_e32 v2, v5 +; GISEL-NEXT: v_mov_b32_e32 v3, v6 +; GISEL-NEXT: .LBB5_7: ; %Flow1 ; GISEL-NEXT: s_or_b64 exec, exec, s[12:13] -; GISEL-NEXT: s_and_saveexec_b64 s[4:5], s[10:11] -; GISEL-NEXT: ; %bb.11: ; %itofp-sw-bb -; GISEL-NEXT: v_lshlrev_b64 v[6:7], 1, v[0:1] -; GISEL-NEXT: ; %bb.12: ; %itofp-sw-epilog +; GISEL-NEXT: .LBB5_8: ; %Flow2 +; GISEL-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11] +; GISEL-NEXT: ; %bb.9: ; %itofp-sw-bb +; GISEL-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1] +; GISEL-NEXT: ; %bb.10: ; %itofp-sw-epilog ; GISEL-NEXT: s_or_b64 exec, exec, s[4:5] -; GISEL-NEXT: v_bfe_u32 v0, v6, 2, 1 -; GISEL-NEXT: v_or_b32_e32 v0, v6, v0 +; GISEL-NEXT: v_bfe_u32 v2, v0, 2, 1 +; GISEL-NEXT: v_or_b32_e32 v0, v0, v2 ; GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 1, v0 -; GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v7, vcc -; GISEL-NEXT: v_lshrrev_b64 v[2:3], 2, v[0:1] -; GISEL-NEXT: v_and_b32_e32 v3, 0x4000000, v0 -; GISEL-NEXT: v_mov_b32_e32 v4, 0 -; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[3:4] +; GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc +; GISEL-NEXT: v_and_b32_e32 v2, 0x4000000, v0 +; GISEL-NEXT: v_mov_b32_e32 v3, 0 +; GISEL-NEXT: v_lshrrev_b64 v[4:5], 2, v[0:1] +; GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3] ; GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc -; GISEL-NEXT: ; %bb.13: ; %itofp-if-then20 -; GISEL-NEXT: v_lshrrev_b64 v[2:3], 3, v[0:1] -; GISEL-NEXT: v_mov_b32_e32 v10, v11 -; GISEL-NEXT: ; %bb.14: ; %Flow +; GISEL-NEXT: ; %bb.11: ; %itofp-if-then20 +; GISEL-NEXT: v_lshrrev_b64 v[4:5], 3, v[0:1] +; GISEL-NEXT: v_mov_b32_e32 v6, v7 +; GISEL-NEXT: ; %bb.12: ; %Flow ; GISEL-NEXT: s_or_b64 exec, exec, s[4:5] -; GISEL-NEXT: .LBB5_15: ; %Flow7 +; GISEL-NEXT: .LBB5_13: ; %Flow4 ; GISEL-NEXT: s_or_b64 exec, exec, s[8:9] -; GISEL-NEXT: v_lshl_add_u32 v0, v10, 23, 1.0 +; GISEL-NEXT: v_lshl_add_u32 v0, v6, 23, 1.0 ; GISEL-NEXT: v_mov_b32_e32 v1, 0x7fffff -; GISEL-NEXT: v_and_or_b32 v0, v2, v1, v0 -; GISEL-NEXT: v_cvt_f16_f32_e32 v2, v0 -; GISEL-NEXT: .LBB5_16: ; %Flow8 +; GISEL-NEXT: v_and_or_b32 v0, v4, v1, v0 +; GISEL-NEXT: v_cvt_f16_f32_e32 v4, v0 +; GISEL-NEXT: .LBB5_14: ; %Flow5 ; GISEL-NEXT: s_or_b64 exec, exec, s[6:7] -; GISEL-NEXT: v_mov_b32_e32 v0, v2 +; GISEL-NEXT: v_mov_b32_e32 v0, v4 ; GISEL-NEXT: s_setpc_b64 s[30:31] %cvt = uitofp i128 %x to half ret half %cvt -- GitLab From c957715d721b70591ef32dd9609d6d96de9f0554 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Fri, 15 Mar 2024 14:12:59 +0000 Subject: [PATCH 013/514] [X86] isGuaranteedNotToBeUndefOrPoisonForTargetNode - generalize shuffle decoding to support more target shuffles in the future. --- llvm/lib/Target/X86/X86ISelLowering.cpp | 32 +++++++++++++++++-------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index dbfcb3752ae8..b9a87f9024c7 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -42665,7 +42665,6 @@ SDValue X86TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode( bool X86TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode( SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool PoisonOnly, unsigned Depth) const { - unsigned EltsBits = Op.getScalarValueSizeInBits(); unsigned NumElts = DemandedElts.getBitWidth(); // TODO: Add more target shuffles. @@ -42673,15 +42672,28 @@ bool X86TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode( case X86ISD::PSHUFD: case X86ISD::VPERMILPI: { SmallVector Mask; - DecodePSHUFMask(NumElts, EltsBits, Op.getConstantOperandVal(1), Mask); - - APInt DemandedSrcElts = APInt::getZero(NumElts); - for (unsigned I = 0; I != NumElts; ++I) - if (DemandedElts[I]) - DemandedSrcElts.setBit(Mask[I]); - - return DAG.isGuaranteedNotToBeUndefOrPoison( - Op.getOperand(0), DemandedSrcElts, PoisonOnly, Depth + 1); + SmallVector Ops; + if (getTargetShuffleMask(Op.getNode(), Op.getSimpleValueType(), true, Ops, + Mask)) { + SmallVector DemandedSrcElts(Ops.size(), + APInt::getZero(NumElts)); + for (auto M : enumerate(Mask)) { + if (!DemandedElts[M.index()] || M.value() == SM_SentinelZero) + continue; + if (M.value() == SM_SentinelUndef) + return false; + assert(0 <= M.value() && M.value() < (int)(Ops.size() * NumElts) && + "Shuffle mask index out of range"); + DemandedSrcElts[M.value() / NumElts].setBit(M.value() % NumElts); + } + for (auto Op : enumerate(Ops)) + if (!DemandedSrcElts[Op.index()].isZero() && + !DAG.isGuaranteedNotToBeUndefOrPoison( + Op.value(), DemandedSrcElts[Op.index()], PoisonOnly, Depth + 1)) + return false; + return true; + } + break; } } return TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode( -- GitLab From bf3f86623c4712bff146b5d168cad5f7e658ac56 Mon Sep 17 00:00:00 2001 From: Jay Foad Date: Fri, 15 Mar 2024 13:39:00 +0000 Subject: [PATCH 014/514] [AMDGPU] Simplify GFX11 and GFX12 FLAT saddr field definition It is simpler to define this field correctly in the base class for the Reals for each architecture, than to override it in subclasses for different addressing modes. --- llvm/lib/Target/AMDGPU/FLATInstructions.td | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/FLATInstructions.td b/llvm/lib/Target/AMDGPU/FLATInstructions.td index 594754492564..c6a0d6e89f44 100644 --- a/llvm/lib/Target/AMDGPU/FLATInstructions.td +++ b/llvm/lib/Target/AMDGPU/FLATInstructions.td @@ -174,7 +174,7 @@ class VFLAT_Real op, FLAT_Pseudo ps, string opName = ps.Mnemonic> : bits<8> vaddr; bits<24> offset; - let Inst{6-0} = !if(ps.enabled_saddr, saddr, 0x7f); + let Inst{6-0} = !if(ps.enabled_saddr, saddr, SGPR_NULL_gfx11plus.Index); let Inst{21-14} = op; let Inst{31-26} = 0x3b; let Inst{39-32} = !if(ps.has_vdst, vdst, ?); @@ -2353,6 +2353,7 @@ class FLAT_Real_gfx11 op, FLAT_Pseudo ps, string opName = ps.Mnemonic> let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glcValue); let Inst{15} = cpol{CPolBit.SLC}; let Inst{17-16} = seg; + let Inst{54-48} = !if(ps.enabled_saddr, saddr, SGPR_NULL_gfx11plus.Index); let Inst{55} = ps.sve; } @@ -2363,15 +2364,11 @@ multiclass FLAT_Aliases_gfx11 { multiclass FLAT_Real_Base_gfx11 op, string ps, string opName, int renamed = false> : FLAT_Aliases_gfx11 { - def _gfx11 : FLAT_Real_gfx11(ps), opName> { - let Inst{54-48} = SGPR_NULL_gfx11plus.Index; - } + def _gfx11 : FLAT_Real_gfx11(ps), opName>; } multiclass FLAT_Real_RTN_gfx11 op, string ps, string opName> { - def _RTN_gfx11 : FLAT_Real_gfx11(ps#"_RTN"), opName> { - let Inst{54-48} = SGPR_NULL_gfx11plus.Index; - } + def _RTN_gfx11 : FLAT_Real_gfx11(ps#"_RTN"), opName>; } multiclass FLAT_Real_SADDR_gfx11 op, string ps, string opName> { @@ -2384,7 +2381,6 @@ multiclass FLAT_Real_SADDR_RTN_gfx11 op, string ps, string opName> { multiclass FLAT_Real_ST_gfx11 op, string ps, string opName> { def _ST_gfx11 : FLAT_Real_gfx11(ps#"_ST"), opName> { - let Inst{54-48} = SGPR_NULL_gfx11plus.Index; let OtherPredicates = [HasFlatScratchSTMode]; } } @@ -2579,15 +2575,11 @@ multiclass VFLAT_Aliases_gfx12 op, string ps = NAME, string opName = !tolower(NAME), int renamed = false, string alias = ""> : VFLAT_Aliases_gfx12 { - def _gfx12 : VFLAT_Real_gfx12(ps), opName> { - let Inst{6-0} = !cast(SGPR_NULL_gfx11plus.HWEncoding); - } + def _gfx12 : VFLAT_Real_gfx12(ps), opName>; } multiclass VFLAT_Real_RTN_gfx12 op, string ps, string opName> { - def _RTN_gfx12 : VFLAT_Real_gfx12(ps#"_RTN"), opName> { - let Inst{6-0} = !cast(SGPR_NULL_gfx11plus.HWEncoding); - } + def _RTN_gfx12 : VFLAT_Real_gfx12(ps#"_RTN"), opName>; } multiclass VFLAT_Real_SADDR_gfx12 op, string ps, string opName> { @@ -2600,7 +2592,6 @@ multiclass VFLAT_Real_SADDR_RTN_gfx12 op, string ps, string opName> { multiclass VFLAT_Real_ST_gfx12 op, string ps, string opName> { def _ST_gfx12 : VFLAT_Real_gfx12(ps#"_ST"), opName> { - let Inst{6-0} = !cast(SGPR_NULL_gfx11plus.HWEncoding); let OtherPredicates = [HasFlatScratchSTMode]; } } -- GitLab From 65284be2992fc7c6feafc44dda7c0f00df7aacfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?= =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?= =?UTF-8?q?=E3=83=B3=29?= Date: Fri, 15 Mar 2024 07:22:22 -0700 Subject: [PATCH 015/514] [flang][cuda] Lower dim3 grid z correctly on calls (#85346) --- flang/lib/Lower/ConvertCall.cpp | 6 ++++-- flang/test/Lower/CUDA/cuda-kernel-calls.cuf | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/flang/lib/Lower/ConvertCall.cpp b/flang/lib/Lower/ConvertCall.cpp index 990912195d14..95569337a06e 100644 --- a/flang/lib/Lower/ConvertCall.cpp +++ b/flang/lib/Lower/ConvertCall.cpp @@ -416,7 +416,7 @@ std::pair Fortran::lower::genCallOpAndResult( mlir::Type i32Ty = builder.getI32Type(); mlir::Value one = builder.createIntegerConstant(loc, i32Ty, 1); - mlir::Value grid_x, grid_y; + mlir::Value grid_x, grid_y, grid_z; if (caller.getCallDescription().chevrons()[0].GetType()->category() == Fortran::common::TypeCategory::Integer) { // If grid is an integer, it is converted to dim3(grid,1,1). Since z is @@ -426,11 +426,13 @@ std::pair Fortran::lower::genCallOpAndResult( fir::getBase(converter.genExprValue( caller.getCallDescription().chevrons()[0], stmtCtx))); grid_y = one; + grid_z = one; } else { auto dim3Addr = converter.genExprAddr( caller.getCallDescription().chevrons()[0], stmtCtx); grid_x = readDim3Value(builder, loc, fir::getBase(dim3Addr), "x"); grid_y = readDim3Value(builder, loc, fir::getBase(dim3Addr), "y"); + grid_z = readDim3Value(builder, loc, fir::getBase(dim3Addr), "z"); } mlir::Value block_x, block_y, block_z; @@ -466,7 +468,7 @@ std::pair Fortran::lower::genCallOpAndResult( caller.getCallDescription().chevrons()[3], stmtCtx))); builder.create( - loc, funcType.getResults(), funcSymbolAttr, grid_x, grid_y, one, + loc, funcType.getResults(), funcSymbolAttr, grid_x, grid_y, grid_z, block_x, block_y, block_z, bytes, stream, operands); callNumResults = 0; } else if (caller.requireDispatchCall()) { diff --git a/flang/test/Lower/CUDA/cuda-kernel-calls.cuf b/flang/test/Lower/CUDA/cuda-kernel-calls.cuf index d5dabaa1df96..55b5246e59a0 100644 --- a/flang/test/Lower/CUDA/cuda-kernel-calls.cuf +++ b/flang/test/Lower/CUDA/cuda-kernel-calls.cuf @@ -20,13 +20,15 @@ contains call dev_kernel0<<<10, 20>>>() ! CHECK: fir.cuda_kernel_launch @_QMtest_callPdev_kernel0<<<%c10{{.*}}, %c1{{.*}}, %c1{{.*}}, %c20{{.*}}, %c1{{.*}}, %c1{{.*}}>>>() - call dev_kernel0<<< __builtin_dim3(1,1), __builtin_dim3(32,1,1) >>> + call dev_kernel0<<< __builtin_dim3(1,1,4), __builtin_dim3(32,1,1) >>> ! CHECK: %[[ADDR_DIM3_GRID:.*]] = fir.address_of(@_QQro._QM__fortran_builtinsT__builtin_dim3.{{.*}}) : !fir.ref> ! CHECK: %[[DIM3_GRID:.*]]:2 = hlfir.declare %[[ADDR_DIM3_GRID]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QQro._QM__fortran_builtinsT__builtin_dim3.0"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) ! CHECK: %[[GRID_X:.*]] = hlfir.designate %[[DIM3_GRID]]#1{"x"} : (!fir.ref>) -> !fir.ref ! CHECK: %[[GRID_X_LOAD:.*]] = fir.load %[[GRID_X]] : !fir.ref ! CHECK: %[[GRID_Y:.*]] = hlfir.designate %[[DIM3_GRID]]#1{"y"} : (!fir.ref>) -> !fir.ref ! CHECK: %[[GRID_Y_LOAD:.*]] = fir.load %[[GRID_Y]] : !fir.ref +! CHECK: %[[GRID_Z:.*]] = hlfir.designate %[[DIM3_GRID]]#1{"z"} : (!fir.ref>) -> !fir.ref +! CHECK: %[[GRID_Z_LOAD:.*]] = fir.load %[[GRID_Z]] : !fir.ref ! CHECK: %[[ADDR_DIM3_BLOCK:.*]] = fir.address_of(@_QQro._QM__fortran_builtinsT__builtin_dim3.{{.*}}) : !fir.ref> ! CHECK: %[[DIM3_BLOCK:.*]]:2 = hlfir.declare %[[ADDR_DIM3_BLOCK]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QQro._QM__fortran_builtinsT__builtin_dim3.1"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) ! CHECK: %[[BLOCK_X:.*]] = hlfir.designate %[[DIM3_BLOCK]]#1{"x"} : (!fir.ref>) -> !fir.ref @@ -35,7 +37,7 @@ contains ! CHECK: %[[BLOCK_Y_LOAD:.*]] = fir.load %[[BLOCK_Y]] : !fir.ref ! CHECK: %[[BLOCK_Z:.*]] = hlfir.designate %[[DIM3_BLOCK]]#1{"z"} : (!fir.ref>) -> !fir.ref ! CHECK: %[[BLOCK_Z_LOAD:.*]] = fir.load %[[BLOCK_Z]] : !fir.ref -! CHECK: fir.cuda_kernel_launch @_QMtest_callPdev_kernel0<<<%[[GRID_X_LOAD]], %[[GRID_Y_LOAD]], %c1{{.*}}, %[[BLOCK_X_LOAD]], %[[BLOCK_Y_LOAD]], %[[BLOCK_Z_LOAD]]>>>() +! CHECK: fir.cuda_kernel_launch @_QMtest_callPdev_kernel0<<<%[[GRID_X_LOAD]], %[[GRID_Y_LOAD]], %[[GRID_Z_LOAD]], %[[BLOCK_X_LOAD]], %[[BLOCK_Y_LOAD]], %[[BLOCK_Z_LOAD]]>>>() call dev_kernel0<<<10, 20, 2>>>() ! CHECK: fir.cuda_kernel_launch @_QMtest_callPdev_kernel0<<<%c10{{.*}}, %c1{{.*}}, %c1{{.*}}, %c20{{.*}}, %c1{{.*}}, %c1{{.*}}, %c2{{.*}}>>>() -- GitLab From 0e0bfacff71859d1f9212205f8f873d47029d3fb Mon Sep 17 00:00:00 2001 From: yonghong-song Date: Fri, 15 Mar 2024 07:24:28 -0700 Subject: [PATCH 016/514] [BPF] Add support for may_goto insn (#85358) Alexei added may_goto insn in [1]. The asm syntax for may_goto looks like may_goto